docstr: Parameters
  ----------
  tokenizer
      The tokenizer
  doc_stride
      The stride to chunk the document
  max_seq_length
      Maximum length of the merged data
  max_query_length
      Maximum query length
docstr_tok: ["Parameters","--","--","--","--","--","tokenizer","The","tokenizer","doc_stride","The","stride","to","chunk","the","document","max_seq_length","Maximum","length","of","the","merged","data","max_query_length","Maximum","query","length"]
code: def __init__(
      self, tokenizer, doc_stride, max_seq_length, max_query_length
  ):
      """
  
      Parameters
      ----------
      tokenizer
          The tokenizer
      doc_stride
          The stride to chunk the document
      max_seq_length
          Maximum length of the merged data
      max_query_length
          Maximum query length
      """
      self._tokenizer = tokenizer
      self._doc_stride = doc_stride
      self._max_seq_length = max_seq_length
      self._max_query_length = max_query_length
  
      vocab = tokenizer.vocab
      self.pad_id = vocab.pad_id
      # For roberta model, taking sepecial token <s> as [CLS] and </s> as [SEP]
      self.cls_id = (
          vocab.bos_id
          if "cls_token" not in vocab.special_token_keys
          else vocab.cls_id
      )
      self.sep_id = (
          vocab.eos_id
          if "sep_token" not in vocab.special_token_keys
          else vocab.sep_id
      )
  
      # TODO(sxjscience) Consider to combine the NamedTuple and batchify functionality.
      self.BatchifyFunction = bf.NamedTuple(
          ChunkFeature,
          {
              "qas_id": bf.List(),
              "data": bf.Pad(val=self.pad_id, round_to=args.round_to),
              "valid_length": bf.Stack(),
              "segment_ids": bf.Pad(round_to=args.round_to),
              "masks": bf.Pad(val=1, round_to=args.round_to),
              "is_impossible": bf.Stack(),
              "gt_start": bf.Stack(),
              "gt_end": bf.Stack(),
              "context_offset": bf.Stack(),
              "chunk_start": bf.Stack(),
              "chunk_length": bf.Stack(),
          },
      )
  
imports: import os
  import json
  import time
  import logging
  import argparse
  import ast
  import functools
  import collections
  import dataclasses
  from dataclasses import dataclass
  from multiprocessing import Pool, cpu_count
  import mxnet
  import numpy
  from mxnet.lr_scheduler import PolyScheduler
  import gluonnlp.data.batchify
  from models import ModelForQABasic, ModelForQAConditionalV1
  from eval_utils import squad_eval
  from squad_utils import SquadFeature, get_squad_examples, convert_squad_example_to_feature
  from gluonnlp.models import get_backbone
  from gluonnlp.utils.misc import repeat, grouper, set_seed, init_comm, logging_config, parse_ctx
  from gluonnlp.initializer import TruncNorm
  from gluonnlp.data.sampler import SplitSampler
  from gluonnlp.utils.parameter import grad_global_norm, clip_grad_global_norm, count_parameters, deduplicate_param_dict
label_desc: Use image node_id 1 to create a new SquadDatasetProcessor object  with example: obj = SquadDatasetProcessor(tokenizer, doc_stride, max_seq_length, max_query_length)