class_docstr: Encapsulates the roberta tokenizer
class_docstr_tok: ["Encapsulates","the","roberta","tokenizer"]
docstr: doc: str, text to be tokenized
docstr_tok: ["doc",":","str",",","text","to","be","tokenized"]
code: def tokenize(self, doc):
      """
      doc: str, text to be tokenized
      """
  
      sents = []
      if self.lower_case:
          doc = doc.lower()
      decoded_text = self.decoder(doc)
      tokens = self.tokenizer.convert_ids_to_tokens(
          self.tokenizer(decoded_text)["input_ids"],
          skip_special_tokens=True,
      )
  
      last_idx = 0
      toks = []
      for raw_token in tokens:
          token_surface_string = self._get_surface_string(raw_token)
          if ord(raw_token[0]) == 266:  # skip new lines
              last_idx += len(raw_token)
              continue
          try:
              token_idx = decoded_text.index(
                  token_surface_string, last_idx
              )
          except Exception as e:
              print(decoded_text)
              print(token_surface_string)
              raise e
          toks.append(
              Tok(
                  _get_pos_tag(token_surface_string),
                  token_surface_string.lower(),
                  raw_token.lower(),
                  ent_type=""
                  if self.entity_type is None
                  else self.entity_type.get(token_surface_string, ""),
                  tag=""
                  if self.tag_type is None
                  else self.tag_type.get(token_surface_string, ""),
                  idx=token_idx,
              )
          )
          last_idx = token_idx + len(token_surface_string)
          if token_surface_string in [
              ".",
              "!",
              "?",
          ]:  # idiot's sentence splitter
              sents.append(toks)
              toks = []
  
      if len(toks) > 0:
          sents.append(toks)
      return Doc(sents, decoded_text)
  
file_path: scattertext/scattertext/tokenizers/transformers.py
imports: import abc
  from abc import ABC
  from scattertext.WhitespaceNLP import Tok, _get_pos_tag, Doc
label_desc: Use image node_id 2 for calling the TransformerTokenizerWrapper obj's underlying member method code with example usage: obj.tokenize(doc) and returns: Doc