code: def token_chunker(
      self,
      question,
      context,
      max_chunk_size=512,
      stride=2,
      max_num_chunks=5,
  ):
      # we tokenize question and context once.
      # if question + context > max chunksize, we break it down into multiple chunks of question +
      # subsets of context with some stride overlap
  
      question_tokens = self.tokenizer.encode(question)
      context_tokens = self.tokenizer.encode(
          context, add_special_tokens=False
      )
  
      chunk_holder = []
      chunk_size = max_chunk_size - len(question_tokens) - 1
      # -1 for the 102 end token we append later
      current_pos = 0
      chunk_count = 0
      while current_pos < len(context_tokens) and current_pos >= 0:
          # we want to cap the number of chunks we create
          if max_num_chunks and chunk_count >= max_num_chunks:
              break
  
          end_point = (
              current_pos + chunk_size
              if (current_pos + chunk_size) < len(context_tokens) - 1
              else len(context_tokens) - 1
          )
          token_chunk = (
              question_tokens
              + context_tokens[current_pos:end_point]
              + [102]
          )
  
          # question type is 0, context type is 1, convert to tf
          token_type_ids = [0] * len(question_tokens) + [1] * (
              len(token_chunk) - len(question_tokens)
          )
          token_type_ids = tf.constant(
              token_type_ids,
              dtype="int32",
              shape=(1, len(token_type_ids)),
          )
  
          # attend to every token
          attention_mask = tf.ones(
              (1, len(token_chunk)), dtype=tf.dtypes.int32
          )
  
          # convert token chunk to tf
          token_chunk = tf.constant(
              token_chunk, dtype="int32", shape=(1, len(token_chunk))
          )
  
          chunk_holder.append(
              {
                  "token_ids": token_chunk,
                  "context": self.tokenizer.decode(
                      context_tokens[current_pos:end_point],
                      skip_special_tokens=True,
                  ),
                  "attention_mask": attention_mask,
                  "token_type_ids": token_type_ids,
              }
          )
          current_pos = current_pos + chunk_size - stride + 1
          chunk_count += 1
  
      return chunk_holder
  
file_path: neuralqa/neuralqa/reader/bertreader.py
imports: from neuralqa.reader import Reader
  import tensorflow
  import numpy
  import time
  import logging
label_desc: Use image node_id 4 for calling the BERTReader obj's underlying member method code with example usage: obj.token_chunker(question, context, max_chunk_size, stride, max_num_chunks) and returns: chunk_holder