code: def convert_examples_to_features(
      examples,
      tokenizer,
      max_seq_length,
      doc_stride,
      max_query_length,
      is_training,
      logger,
      cls_token_at_end=False,
      cls_token="[CLS]",
      sep_token="[SEP]",
      pad_token=0,
      sequence_a_segment_id=0,
      sequence_b_segment_id=1,
      cls_token_segment_id=0,
      pad_token_segment_id=0,
      mask_padding_with_zero=True,
  ):
      """Loads a data file into a list of `InputBatch`s."""
  
      unique_id = 1000000000
      # cnt_pos, cnt_neg = 0, 0
      # max_N, max_M = 1024, 1024
      # f = np.zeros((max_N, max_M), dtype=np.float32)
  
      features = []
      for example_index, example in enumerate(examples):
          if example_index % 10000 == 0:
              if logger:
                  logger.info(
                      "Writing example {} of {}".format(
                          example_index, len(examples)
                      )
                  )
  
          # if example_index % 100 == 0:
          #     logger.info('Converting %s/%s pos %s neg %s', example_index, len(examples), cnt_pos, cnt_neg)
  
          query_tokens = tokenizer.tokenize(example.question_text)
  
          if len(query_tokens) > max_query_length:
              query_tokens = query_tokens[0:max_query_length]
  
          tok_to_orig_index = []
          orig_to_tok_index = []
          all_doc_tokens = []
          for i, token in enumerate(example.doc_tokens):
              orig_to_tok_index.append(len(all_doc_tokens))
              sub_tokens = tokenizer.tokenize(token)
              for sub_token in sub_tokens:
                  tok_to_orig_index.append(i)
                  all_doc_tokens.append(sub_token)
  
          tok_start_position = None
          tok_end_position = None
          if is_training and example.is_impossible:
              tok_start_position = -1
              tok_end_position = -1
          if is_training and not example.is_impossible:
              tok_start_position = orig_to_tok_index[
                  example.start_position
              ]
              if example.end_position < len(example.doc_tokens) - 1:
                  tok_end_position = (
                      orig_to_tok_index[example.end_position + 1] - 1
                  )
              else:
                  tok_end_position = len(all_doc_tokens) - 1
              (
                  tok_start_position,
                  tok_end_position,
              ) = _improve_answer_span(
                  all_doc_tokens,
                  tok_start_position,
                  tok_end_position,
                  tokenizer,
                  example.orig_answer_text,
              )
  
          # The -3 accounts for [CLS], [SEP] and [SEP]
          max_tokens_for_doc = max_seq_length - len(query_tokens) - 3
  
          # We can have documents that are longer than the maximum sequence length.
          # To deal with this we do a sliding window approach, where we take chunks
          # of the up to our max length with a stride of `doc_stride`.
          _DocSpan = (
              collections.namedtuple(  # pylint: disable=invalid-name
                  "DocSpan", ["start", "length"]
              )
          )
          doc_spans = []
          start_offset = 0
          while start_offset < len(all_doc_tokens):
              length = len(all_doc_tokens) - start_offset
              if length > max_tokens_for_doc:
                  length = max_tokens_for_doc
              doc_spans.append(
                  _DocSpan(start=start_offset, length=length)
              )
              if start_offset + length == len(all_doc_tokens):
                  break
              start_offset += min(length, doc_stride)
  
          for doc_span_index, doc_span in enumerate(doc_spans):
              tokens = []
              token_to_orig_map = {}
              token_is_max_context = {}
              segment_ids = []
  
              # p_mask: mask with 1 for token than cannot be in the answer (0 for token which can be in an answer)
              # Original TF implem also keep the classification token (set to 0) (not sure why...)
              p_mask = []
  
              # CLS token at the beginning
              if not cls_token_at_end:
                  tokens.append(cls_token)
                  segment_ids.append(cls_token_segment_id)
                  p_mask.append(0)
                  cls_index = 0
  
              # Query
              for token in query_tokens:
                  tokens.append(token)
                  segment_ids.append(sequence_a_segment_id)
                  p_mask.append(1)
  
              # SEP token
              tokens.append(sep_token)
              segment_ids.append(sequence_a_segment_id)
              p_mask.append(1)
  
              # Paragraph
              for i in range(doc_span.length):
                  split_token_index = doc_span.start + i
                  token_to_orig_map[len(tokens)] = tok_to_orig_index[
                      split_token_index
                  ]
  
                  is_max_context = _check_is_max_context(
                      doc_spans, doc_span_index, split_token_index
                  )
                  token_is_max_context[len(tokens)] = is_max_context
                  tokens.append(all_doc_tokens[split_token_index])
                  segment_ids.append(sequence_b_segment_id)
                  p_mask.append(0)
              paragraph_len = doc_span.length
  
              # SEP token
              tokens.append(sep_token)
              segment_ids.append(sequence_b_segment_id)
              p_mask.append(1)
  
              # CLS token at the end
              if cls_token_at_end:
                  tokens.append(cls_token)
                  segment_ids.append(cls_token_segment_id)
                  p_mask.append(0)
                  cls_index = (
                      len(tokens) - 1
                  )  # Index of classification token
  
              input_ids = tokenizer.convert_tokens_to_ids(tokens)
  
              # The mask has 1 for real tokens and 0 for padding tokens. Only real
              # tokens are attended to.
              input_mask = [1 if mask_padding_with_zero else 0] * len(
                  input_ids
              )
  
              # Zero-pad up to the sequence length.
              while len(input_ids) < max_seq_length:
                  input_ids.append(pad_token)
                  input_mask.append(0 if mask_padding_with_zero else 1)
                  segment_ids.append(pad_token_segment_id)
                  p_mask.append(1)
  
              assert len(input_ids) == max_seq_length
              assert len(input_mask) == max_seq_length
              assert len(segment_ids) == max_seq_length
  
              span_is_impossible = example.is_impossible
              start_position = None
              end_position = None
              if is_training and not span_is_impossible:
                  # For training, if our document chunk does not contain an annotation
                  # we throw it out, since there is nothing to predict.
                  doc_start = doc_span.start
                  doc_end = doc_span.start + doc_span.length - 1
                  out_of_span = False
                  if not (
                      tok_start_position >= doc_start
                      and tok_end_position <= doc_end
                  ):
                      out_of_span = True
                  if out_of_span:
                      start_position = 0
                      end_position = 0
                      span_is_impossible = True
                  else:
                      doc_offset = len(query_tokens) + 2
                      start_position = (
                          tok_start_position - doc_start + doc_offset
                      )
                      end_position = (
                          tok_end_position - doc_start + doc_offset
                      )
  
              if is_training and span_is_impossible:
                  start_position = cls_index
                  end_position = cls_index
  
              features.append(
                  InputFeatures(
                      unique_id=unique_id,
                      example_index=example_index,
                      doc_span_index=doc_span_index,
                      tokens=tokens,
                      token_to_orig_map=token_to_orig_map,
                      token_is_max_context=token_is_max_context,
                      input_ids=input_ids,
                      input_mask=input_mask,
                      segment_ids=segment_ids,
                      cls_index=cls_index,
                      p_mask=p_mask,
                      paragraph_len=paragraph_len,
                      start_position=start_position,
                      end_position=end_position,
                      is_impossible=span_is_impossible,
                  )
              )
              unique_id += 1
  
      return features
  
imports: import json
  import os
  import torch
  from pathlib import Path
  import pickle
  import logging
  import collections
  from io import open
  import shutil
  from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
  from torch.utils.data.distributed import DistributedSampler
  from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize
  from transformers import WEIGHTS_NAME, BertTokenizer, XLNetTokenizer, DistilBertTokenizer
label_desc: Use image node_id 2 for calling a global function with example usage: convert_examples_to_features(examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training, logger, cls_token_at_end, cls_token, sep_token, pad_token, sequence_a_segment_id, sequence_b_segment_id, cls_token_segment_id, pad_token_segment_id, mask_padding_with_zero) and returns: features