docstr: This function convert raw inputs into tokens, each token is represented
  by token(str), start and end indices in the raw inputs. There are two
  possible inputs to this function depends if the tokenized in implemented
  in TorchScript or not.
  
  Case 1: Tokenizer has a full TorchScript implementation, the input will
  be a list of sentences (in most case it is single sentence or a pair).
  
  Case 2: Tokenizer have partial or no TorchScript implementation, in most
  case, the tokenizer will be host in Yoda, the input will be a list of
  pre-processed tokens.
  
  Returns:
      per_sentence_tokens: tokens per sentence level, each token is
      represented by token(str), start and end indices.
code: def tokenize(
      self,
      row_text: Optional[List[str]],
      row_pre_tokenized: Optional[List[List[str]]],
  ) -> List[List[Tuple[str, int, int]]]:
      """
      This function convert raw inputs into tokens, each token is represented
      by token(str), start and end indices in the raw inputs. There are two
      possible inputs to this function depends if the tokenized in implemented
      in TorchScript or not.
  
      Case 1: Tokenizer has a full TorchScript implementation, the input will
      be a list of sentences (in most case it is single sentence or a pair).
  
      Case 2: Tokenizer have partial or no TorchScript implementation, in most
      case, the tokenizer will be host in Yoda, the input will be a list of
      pre-processed tokens.
  
      Returns:
          per_sentence_tokens: tokens per sentence level, each token is
          represented by token(str), start and end indices.
      """
      per_sentence_tokens: List[List[Tuple[str, int, int]]] = []
  
      if row_text is not None:
          for text in row_text:
              per_sentence_tokens.append(self.tokenizer.tokenize(text))
      elif row_pre_tokenized is not None:
          for sentence_pre_tokenized in row_pre_tokenized:
              sentence_tokens: List[Tuple[str, int, int]] = []
              for token in sentence_pre_tokenized:
                  sentence_tokens.extend(self.tokenizer.tokenize(token))
              per_sentence_tokens.append(sentence_tokens)
  
      return per_sentence_tokens
  
imports: from typing import Any, Dict, List, Optional, Tuple
  import torch
  from fairseq.data.dictionary import Dictionary
  from fairseq.data.legacy.masked_lm_dictionary import BertDictionary
  from pytext import resources
  from pytext.common.constants import Token
  from pytext.config.component import ComponentType, create_component
  from pytext.data.tensorizers import Tensorizer, TensorizerScriptImpl
  from pytext.data.tokenizers import Tokenizer, WordPieceTokenizer
  from pytext.data.utils import BOS, EOS, MASK, PAD, UNK, Vocabulary
  from pytext.torchscript.tensorizer.tensorizer import VocabLookup
  from pytext.torchscript.utils import pad_2d_mask, ScriptBatchInput
  from pytext.torchscript.vocab import ScriptVocabulary
  from pytext.utils.file_io import PathManager
  from pytext.utils.lazy import lazy_property
  from pytorch.text.fb.utils.transform_utils import pad_2d
label_desc: Use image node_id 6 for calling the BERTTensorizerBaseScriptImpl obj's underlying member method code with example usage: obj.tokenize(row_text, row_pre_tokenized) and returns: per_sentence_tokens