docstr: Process the data to the following format.
  
  Note that we mask all the special tokens except the CLS token. The reason for not masking
  the CLS token is that if the question is not answerable, we will set the start and end to
  be 0.
  
  
  Merged:      <CLS> Question <SEP> Context <SEP>
  Segment IDs:  0       0       0      1      1
  Mask:         0       1       1      0      1
  
  Here, we need to emphasize that when mask = 1, the data are actually not masked!
  
  Parameters
  ----------
  feature
      Tokenized SQuAD feature
  
  Returns
  -------
  ret
      Divide the feature into multiple chunks and extract the feature which contains
      the following:
      - data
          The data that concatenates the query and the context + special tokens
      - valid_length
          The valid_length of the data
      - segment_ids
          We assign the query part as segment 0 and the context part as segment 1.
      - masks
          We mask all the special tokens. 1 --> not masked, 0 --> masked.
      - is_impossible
          Whether the provided context is impossible to answer or not.
      - gt_start
          The ground-truth start location of the span
      - gt_end
          The ground-truth end location of the span
      - chunk_start
          The start of the chunk
      - chunk_length
          The length of the chunk
docstr_tok: ["Process","the","data","to","the","following","format",".","Note","that","we","mask","all","the","special","tokens","except","the","CLS","token",".","The","reason","for","not","masking","the","CLS","token","is","that","if","the","question","is","not","answerable",",","we","will","set","the","start","and","end","to","be","0",".","Merged",":","<","CLS",">","Question","<","SEP",">","Context","<","SEP",">","Segment","IDs",":","0","0","0","1","1","Mask",":","0","1","1","0","1","Here",",","we","need","to","emphasize","that","when","mask","=","1",",","the","data","are","actually","not","masked","!","Parameters","--","--","--","--","--","feature","Tokenized","SQuAD","feature","Returns","--","--","--","-","ret","Divide","the","feature","into","multiple","chunks","and","extract","the","feature","which","contains","the","following",":","-","data","The","data","that","concatenates","the","query","and","the","context","+","special","tokens","-","valid_length","The","valid_length","of","the","data","-","segment_ids","We","assign","the","query","part","as","segment","0","and","the","context","part","as","segment","1",".","-","masks","We","mask","all","the","special","tokens",".","1","--",">","not","masked",",","0","--",">","masked",".","-","is_impossible","Whether","the","provided","context","is","impossible","to","answer","or","not",".","-","gt_start","The","ground-truth","start","location","of","the","span","-","gt_end","The","ground-truth","end","location","of","the","span","-","chunk_start","The","start","of","the","chunk","-","chunk_length","The","length","of","the","chunk"]
code: def process_sample(self, feature: SquadFeature):
      """Process the data to the following format.
  
      Note that we mask all the special tokens except the CLS token. The reason for not masking
      the CLS token is that if the question is not answerable, we will set the start and end to
      be 0.
  
  
      Merged:      <CLS> Question <SEP> Context <SEP>
      Segment IDs:  0       0       0      1      1
      Mask:         0       1       1      0      1
  
      Here, we need to emphasize that when mask = 1, the data are actually not masked!
  
      Parameters
      ----------
      feature
          Tokenized SQuAD feature
  
      Returns
      -------
      ret
          Divide the feature into multiple chunks and extract the feature which contains
          the following:
          - data
              The data that concatenates the query and the context + special tokens
          - valid_length
              The valid_length of the data
          - segment_ids
              We assign the query part as segment 0 and the context part as segment 1.
          - masks
              We mask all the special tokens. 1 --> not masked, 0 --> masked.
          - is_impossible
              Whether the provided context is impossible to answer or not.
          - gt_start
              The ground-truth start location of the span
          - gt_end
              The ground-truth end location of the span
          - chunk_start
              The start of the chunk
          - chunk_length
              The length of the chunk
      """
      ret = []
      truncated_query_ids = feature.query_token_ids[
          : self._max_query_length
      ]
      chunks = feature.get_chunks(
          doc_stride=self._doc_stride,
          max_chunk_length=self._max_seq_length
          - len(truncated_query_ids)
          - 3,
      )
      for chunk in chunks:
          data = np.array(
              [self.cls_id]
              + truncated_query_ids
              + [self.sep_id]
              + feature.context_token_ids[
                  chunk.start : (chunk.start + chunk.length)
              ]
              + [self.sep_id],
              dtype=np.int32,
          )
          valid_length = len(data)
          segment_ids = np.array(
              [0]
              + [0] * len(truncated_query_ids)
              + [0]
              + [1] * chunk.length
              + [1],
              dtype=np.int32,
          )
          masks = np.array(
              [0]
              + [1] * len(truncated_query_ids)
              + [1]
              + [0] * chunk.length
              + [1],
              dtype=np.int32,
          )
          context_offset = len(truncated_query_ids) + 2
          if chunk.gt_start_pos is None and chunk.gt_end_pos is None:
              start_pos = 0
              end_pos = 0
          else:
              # Here, we increase the start and end because we put query before context
              start_pos = chunk.gt_start_pos + context_offset
              end_pos = chunk.gt_end_pos + context_offset
          chunk_feature = ChunkFeature(
              qas_id=feature.qas_id,
              data=data,
              valid_length=valid_length,
              segment_ids=segment_ids,
              masks=masks,
              is_impossible=chunk.is_impossible,
              gt_start=start_pos,
              gt_end=end_pos,
              context_offset=context_offset,
              chunk_start=chunk.start,
              chunk_length=chunk.length,
          )
          ret.append(chunk_feature)
      return ret
  
imports: import os
  import json
  import time
  import logging
  import argparse
  import ast
  import functools
  import collections
  import dataclasses
  from dataclasses import dataclass
  from multiprocessing import Pool, cpu_count
  from neural_compressor.experimental import Quantization
  import mxnet
  import numpy
  from mxnet.lr_scheduler import PolyScheduler
  import gluonnlp.data.batchify
  from models import ModelForQABasic, ModelForQAConditionalV1
  from eval_utils import squad_eval
  from squad_utils import SquadFeature, get_squad_examples, convert_squad_example_to_feature, get_squad_examples_from_json
  from gluonnlp.models import get_backbone
  from gluonnlp.utils.misc import repeat, grouper, set_seed, init_comm, logging_config, parse_ctx
  from gluonnlp.initializer import TruncNorm
  from gluonnlp.data.sampler import SplitSampler
  from gluonnlp.utils.parameter import grad_global_norm, clip_grad_global_norm, count_parameters, deduplicate_param_dict
  import custom_strategy
label_desc: Use image node_id 2 for calling the SquadDatasetProcessor obj's underlying member method code with example usage: obj.process_sample(feature) and returns: ret