code: def sample_to_features_qa_Natural_Questions(
      sample,
      tokenizer,
      max_seq_len,
      sp_toks_start,
      sp_toks_mid,
      sp_toks_end,
      answer_type_list=None,
      max_answers=6,
  ):
      """Prepares data for processing by the model. Supports cases where there are
      multiple answers for the one question/document pair. max_answers is by default set to 6 since
      that is the most number of answers in the squad2.0 dev set.
  
      :param sample: A Sample object that contains one question / passage pair
      :type sample: Sample
      :param tokenizer: A Tokenizer object
      :type tokenizer: Tokenizer
      :param max_seq_len: The maximum sequence length
      :type max_seq_len: int
      :param sp_toks_start: The number of special tokens that come before the question tokens
      :type sp_toks_start: int
      :param sp_toks_mid: The number of special tokens that come between the question and passage tokens
      :type sp_toks_mid: int
      :param answer_type_list: A list of all the answer types that can be expected e.g. ["no_answer", "span", "yes", "no"] for Natural Questions
      :type answer_type_list: List[str]
      :param max_answers: The maximum number of answer annotations for a sample (In SQuAD, this is 6 hence the default)
      :type max_answers: int
      :return: dict (keys: [input_ids, padding_mask, segment_ids, answer_type_ids, passage_start_t, start_of_word, labels, id, seq_2_start_2])
      """
  
      # Initialize some basic variables
      question_tokens = sample.tokenized["question_tokens"]
      question_start_of_word = sample.tokenized[
          "question_start_of_word"
      ]
      question_len_t = len(question_tokens)
      passage_start_t = sample.tokenized["passage_start_t"]
      passage_tokens = sample.tokenized["passage_tokens"]
      passage_start_of_word = sample.tokenized["passage_start_of_word"]
      passage_len_t = len(passage_tokens)
      answers = sample.tokenized["answers"]
      sample_id = [int(x) for x in sample.id.split("-")]
  
      # Generates a numpy array of shape (max_answers, 2) where (i, 2) indexes into the start and end indices
      # of the ith answer. The array is filled with -1 since the number of answers is often less than max_answers
      # no answer labels are represented by (0,0)
      labels, answer_types = generate_labels(
          answers,
          passage_len_t,
          question_len_t,
          max_answers,
          sp_toks_start,
          sp_toks_mid,
          answer_type_list,
      )
  
      # Generate a start of word vector for the full sequence (i.e. question + answer + special tokens).
      # This will allow us to perform evaluation during training without clear text.
      # Note that in the current implementation, special tokens do not count as start of word.
      start_of_word = combine_vecs(
          question_start_of_word,
          passage_start_of_word,
          tokenizer,
          spec_tok_val=0,
      )
  
      # Combines question_tokens and passage_tokens (str) into a single encoded vector of token indices (int)
      # called input_ids. This encoded vector also contains special tokens (e.g. [CLS]). It will have length =
      # (question_len_t + passage_len_t + n_special_tokens). This may be less than max_seq_len but will not be greater
      # than max_seq_len since truncation was already performed when the document was chunked into passages
      # (c.f. create_samples_squad() )
  
      if tokenizer.is_fast:
          # Detokenize input as fast tokenizer can't handle tokenized input
          question_tokens = " ".join(question_tokens)
          question_tokens = re.sub(r"(^|\s)(##)", "", question_tokens)
          passage_tokens = " ".join(passage_tokens)
          passage_tokens = re.sub(r"(^|\s)(##)", "", passage_tokens)
  
          encoded = tokenizer(
              text=question_tokens,
              text_pair=passage_tokens,
              add_special_tokens=True,
              return_special_tokens_mask=True,
              return_token_type_ids=True,
          )
  
          n_tokens_encoded = len(encoded["input_ids"]) - encoded[
              "special_tokens_mask"
          ].count(1)
          n_tokens_with_metadata = len(
              sample.tokenized["question_tokens"]
          ) + len(sample.tokenized["passage_tokens"])
  
          if n_tokens_encoded != n_tokens_with_metadata:
              tokens_encoded = tokenizer.convert_ids_to_tokens(
                  encoded["input_ids"]
              )
              logger.error(
                  f"FastTokenizer encoded sample to {n_tokens_encoded} tokens,"
                  f" while the previous tokenize_with_metadata produced {n_tokens_with_metadata} tokens. \n"
                  f"Further processing is likely to be wrong.\n"
                  f"FastTokenizer: {tokens_encoded} \n"
                  f"tokenize_with_metadata: {sample.tokenized['question_tokens'] + sample.tokenized['passage_tokens']}"
              )
      else:
          encoded = tokenizer.encode_plus(
              text=sample.tokenized["question_tokens"],
              text_pair=sample.tokenized["passage_tokens"],
              add_special_tokens=True,
              truncation=False,
              truncation_strategy="do_not_truncate",
              return_token_type_ids=True,
              return_tensors=None,
          )
  
      input_ids = encoded["input_ids"]
      segment_ids = encoded["token_type_ids"]
  
      # seq_2_start_t is the index of the first token in the second text sequence (e.g. passage)
      if tokenizer.__class__.__name__ in [
          "RobertaTokenizer",
          "XLMRobertaTokenizer",
      ]:
          seq_2_start_t = get_roberta_seq_2_start(input_ids)
      elif tokenizer.__class__.__name__ == "CamembertTokenizer":
          seq_2_start_t = get_camembert_seq_2_start(input_ids)
      else:
          seq_2_start_t = segment_ids.index(1)
  
      # The mask has 1 for real tokens and 0 for padding tokens. Only real
      # tokens are attended to.
      padding_mask = [1] * len(input_ids)
  
      # The passage mask has 1 for tokens that are valid start or ends for QA spans.
      # 0s are assigned to question tokens, mid special tokens, end special tokens and padding
      # Note that start special tokens are assigned 1 since they can be chosen for a no_answer prediction
      span_mask = [1] * sp_toks_start
      span_mask += [0] * question_len_t
      span_mask += [0] * sp_toks_mid
      span_mask += [1] * passage_len_t
      span_mask += [0] * sp_toks_end
  
      # Pad up to the sequence length. For certain models, the pad token id is not 0 (e.g. Roberta where it is 1)
      pad_idx = tokenizer.pad_token_id
      padding = [pad_idx] * (max_seq_len - len(input_ids))
      zero_padding = [0] * (max_seq_len - len(input_ids))
  
      input_ids += padding
      padding_mask += zero_padding
      segment_ids += zero_padding
      start_of_word += zero_padding
      span_mask += zero_padding
  
      # The XLM-Roberta tokenizer generates a segment_ids vector that separates the first sequence from the second.
      # However, when this is passed in to the forward fn of the Roberta model, it throws an error since
      # Roberta has only a single token embedding (!!!). To get around this, we want to have a segment_ids
      # vec that is only 0s
      if tokenizer.__class__.__name__ in [
          "XLMRobertaTokenizer",
          "RobertaTokenizer",
      ]:
          segment_ids = list(np.zeros_like(segment_ids))
  
      # The first of the labels will be used in train, and the full array will be used in eval.
      # start of word and spec_tok_mask are not actually needed by model.forward() but are needed for model.formatted_preds()
      # passage_start_t is index of passage's first token relative to document
      feature_dict = {
          "input_ids": input_ids,
          "padding_mask": padding_mask,
          "segment_ids": segment_ids,
          "answer_type_ids": answer_types,
          "passage_start_t": passage_start_t,
          "start_of_word": start_of_word,
          "labels": labels,
          "id": sample_id,
          "seq_2_start_t": seq_2_start_t,
          "span_mask": span_mask,
      }
      return [feature_dict]
  
file_path: FARM/farm/data_handler/nq_utils.py
imports: import logging
  import re
  import numpy
  from farm.data_handler.samples import Sample
label_desc: Use image node_id 1 for calling a global function with example usage: sample_to_features_qa_Natural_Questions(sample, tokenizer, max_seq_len, sp_toks_start, sp_toks_mid, sp_toks_end, answer_type_list, max_answers) and returns: list