class_docstr: Version of parlai.utils.bpe.Gpt2BpeHelper that can be TorchScripted.
class_docstr_tok: ["Version","of","parlai.utils.bpe.Gpt2BpeHelper","that","can","be","TorchScripted","."]
docstr: Tokenize text.
  
  Checks for add_prefix_space; handles accordingly.
  
  :param text:
      text to tokenize
  
  :return tokens:
      A list of tokens
docstr_tok: ["Tokenize","text",".","Checks","for","add_prefix_space",";","handles","accordingly",".",":","param","text",":","text","to","tokenize",":","return","tokens",":","A","list","of","tokens"]
code: def encode(self, text: str) -> List[str]:
      """
      Tokenize text.
  
      Checks for add_prefix_space; handles accordingly.
  
      :param text:
          text to tokenize
  
      :return tokens:
          A list of tokens
      """
      if self.add_prefix_space:
          text = f" {text}"
  
      # constants for readability
      FINAL = 1
      SPLITABLE = 0
      pieces: List[Tuple[str, int]] = [(text, SPLITABLE)]
  
      for special_token in self._special_tokens.keys():
          i = 0
          while i < len(pieces):
              subtext, status = pieces[i]
              if status == FINAL:
                  i += 1
                  continue
              split = subtext.split(special_token)
              if len(split) > 1:
                  # special token detected, replace the chunk with small subchunks
                  # split by the special token
                  pieces.pop(i)
                  for j, piece in enumerate(split):
                      if j > 0:
                          # add the special token as a delimiter
                          pieces.insert(
                              i + (2 * j) - 1, (special_token, FINAL)
                          )
                      pieces.insert(i + (2 * j), (piece, SPLITABLE))
              else:
                  i += 1
  
      output: List[str] = []
      for piece, state in pieces:
          if state is FINAL:
              output.append(piece)
          else:
              output += self.helper_encode(piece)
      text = "".join(output)
  
      return output
  
imports: from typing import Dict, List, Tuple
  import torch.jit
label_desc: Use image node_id 3 for calling the ScriptableGpt2BpeHelper obj's underlying member method code with example usage: obj.encode(text) and returns: output