class_docstr: Produces inputs and answer spans for Squad.
class_docstr_tok: ["Produces","inputs","and","answer","spans","for","Squad","."]
code: def numberize(self, row):
      assert len(self.vocab) == len(self.ques_tensorizer.vocab)
      assert len(self.vocab) == len(self.doc_tensorizer.vocab)
  
      # Do NOT use self._lookup_tokens() because it won't enforce max_ques_seq_len.
      ques_tokens, _, _ = self.ques_tensorizer._lookup_tokens(
          row[self.ques_column]
      )
  
      # Start and end indices are those of the tokens in original text.
      # The behavior doesn't change for WordPieceTokenizer because...
      # If there's a word piece, say, "##ly" then the start and end indices
      # will be that of "ly" in original text. These are also char level.
      (
          doc_tokens,
          orig_start_idx,
          orig_end_idx,
      ) = self.doc_tensorizer._lookup_tokens(row[self.doc_column])
  
      # Now map original character level answer spans to token level spans
      start_idx_map = {}
      end_idx_map = {}
      for token_idx, (start_idx, end_idx) in enumerate(
          zip(orig_start_idx, orig_end_idx)
      ):
          start_idx_map[start_idx] = token_idx
          end_idx_map[end_idx] = token_idx
  
      answer_start_token_indices = [
          start_idx_map.get(raw_idx, self.SPAN_PAD_IDX)
          for raw_idx in row[self.answer_starts_column]
      ]
      answer_end_token_indices = [
          end_idx_map.get(raw_idx + len(answer), self.SPAN_PAD_IDX)
          for raw_idx, answer in zip(
              row[self.answer_starts_column], row[self.answers_column]
          )
      ]  # The end index is inclusive. Span = doc_tokens[start:end+1]
  
      if (
          not (answer_start_token_indices and answer_end_token_indices)
          or self._only_pad(answer_start_token_indices)
          or self._only_pad(answer_end_token_indices)
      ):
          answer_start_token_indices = [self.SPAN_PAD_IDX]
          answer_end_token_indices = [self.SPAN_PAD_IDX]
  
      return (
          doc_tokens,
          len(doc_tokens),
          ques_tokens,
          len(ques_tokens),
          answer_start_token_indices,
          answer_end_token_indices,
      )
  
imports: from typing import List
  import torch
  from pytext.common.constants import SpecialTokens
  from pytext.config.component import ComponentType, create_component
  from pytext.data.tensorizers import TokenTensorizer
  from pytext.data.tokenizers import Tokenizer, WordPieceTokenizer
  from pytext.data.utils import pad_and_tensorize, VocabBuilder, Vocabulary
label_desc: Use image node_id 5 for calling the SquadTensorizer obj's underlying member method code with example usage: obj.numberize(row) and returns: doc_tokens,  len,  ques_tokens,  len,  answer_start_token_indices,  answer_end_token_indices