code: def tokenize(
      self,
      text_row: Optional[List[str]],
      token_row: Optional[List[List[str]]],
      language_row: List[str],
  ) -> Tuple[
      List[List[Tuple[str, int, int]]], List[List[Tuple[str, int, int]]]
  ]:
      per_sentence_tokens: List[List[Tuple[str, int, int]]] = []
      per_sentence_languages: List[List[Tuple[str, int, int]]] = []
  
      if text_row is not None:
          """
          Tokenize every single text into a list of tokens.
          For example:
          text_row = ["hello world", "this is sentence"]
          per_sentence_tokens = [["hello", "world"], ["this", "is", "sentence"]]
          """
          for idx, text in enumerate(text_row):
              sentence_tokens: List[
                  Tuple[str, int, int]
              ] = self.tokenizer.tokenize(text)
              sentence_languages: List[Tuple[str, int, int]] = [
                  (language_row[idx], token[1], token[2])
                  for token in sentence_tokens
              ]
  
              per_sentence_tokens.append(sentence_tokens)
              per_sentence_languages.append(sentence_languages)
      elif token_row is not None:
          """
          Tokenize every single token into a sub tokens. (example: BPE)
          For example:
          token_row = [["hello", "world"], ["this", "is", "sentence"]]
          per_sentence_tokens = [
              ["he", "llo" "wo", "rld"], ["th", "is", "is", "sen", "tence"]
          ]
          """
          for idx, sentence_raw_tokens in enumerate(token_row):
              sentence_tokens: List[Tuple[str, int, int]] = []
              sentence_languages: List[Tuple[str, int, int]] = []
  
              for raw_token in sentence_raw_tokens:
                  sub_tokens: List[
                      Tuple[str, int, int]
                  ] = self.tokenizer.tokenize(raw_token)
                  sub_languages: List[Tuple[str, int, int]] = [
                      (language_row[idx], token[1], token[2])
                      for token in sub_tokens
                  ]
  
                  sentence_tokens.extend(sub_tokens)
                  sentence_languages.extend(sub_languages)
  
              per_sentence_tokens.append(sentence_tokens)
              per_sentence_languages.append(sentence_languages)
  
      return per_sentence_tokens, per_sentence_languages
  
file_path: pytext/pytext/torchscript/tensorizer/xlm.py
imports: from typing import List, Optional, Tuple
  import torch
  from pytext.torchscript.utils import pad_2d_mask
  from pytext.torchscript.vocab import ScriptVocabulary
  from .tensorizer import ScriptTensorizer, VocabLookup
label_desc: Use image node_id 2 for calling the ScriptXLMTensorizer obj's underlying member method code with example usage: obj.tokenize(text_row, token_row, language_row) and returns: per_sentence_tokens,  per_sentence_languages