code: def tokenize(self, text):
      # We don't treat new lines as tokens.
      clean_text = text.replace("\n", " ")
      tokens = self.nlp.tokenizer(clean_text)
      if any([p in self.annotators for p in ["lemma", "pos", "ner"]]):
          self.nlp.tagger(tokens)
      if "ner" in self.annotators:
          self.nlp.entity(tokens)
  
      data = []
      for i in range(len(tokens)):
          # Get whitespace
          start_ws = tokens[i].idx
          if i + 1 < len(tokens):
              end_ws = tokens[i + 1].idx
          else:
              end_ws = tokens[i].idx + len(tokens[i].text)
  
          data.append(
              (
                  tokens[i].text,
                  text[start_ws:end_ws],
                  (tokens[i].idx, tokens[i].idx + len(tokens[i].text)),
                  tokens[i].tag_,
                  tokens[i].lemma_,
                  tokens[i].ent_type_,
              )
          )
  
      # Set special option for non-entity tag: '' vs 'O' in spaCy
      return Tokens(data, self.annotators, opts={"non_ent": ""})
  
imports: import copy
  import logging
  import regex
  import spacy
label_desc: Use image node_id 2 for calling the SpacyTokenizer obj's underlying member method code with example usage: obj.tokenize(text) and returns: Tokens