class_docstr: Creates features for entity extraction.
class_docstr_tok: ["Creates","features","for","entity","extraction","."]
code: def tokenize(self, message: Message, attribute: Text) -> List[Token]:
      text = message.get(attribute)
  
      # we need to use regex instead of re, because of
      # https://stackoverflow.com/questions/12746458/python-unicode-regular-expression-matching-failing-with-some-unicode-characters
  
      # remove 'not a word character' if
      words = regex.sub(
          # there is a space or an end of a string after it
          r"[^\w#@&]+(?=\s|$)|"
          # there is a space or beginning of a string before it
          # not followed by a number
          r"(\s|^)[^\w#@&]+(?=[^0-9\s])|"
          # not in between numbers and not . or @ or & or - or #
          # e.g. 10'000.00 or blabla@gmail.com
          # and not url characters
          r"(?<=[^0-9\s])[^\w._~:/?#\[\]()@!$&*+,;=-]+(?=[^0-9\s])",
          " ",
          text,
      ).split()
  
      words = [self.remove_emoji(w) for w in words]
      words = [w for w in words if w]
  
      # if we removed everything like smiles `:)`, use the whole text as 1 token
      if not words:
          words = [text]
  
      tokens = self._convert_words_to_tokens(words, text)
  
      return self._apply_token_pattern(tokens)
  
imports: from __future__ import annotations
  from typing import Any, Dict, List, Optional, Text
  import regex
  import rasa.shared.utils.io
  import rasa.utils.io
  from rasa.engine.graph import ExecutionContext
  from rasa.engine.recipes.default_recipe import DefaultV1Recipe
  from rasa.engine.storage.resource import Resource
  from rasa.engine.storage.storage import ModelStorage
  from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
  from rasa.shared.constants import DOCS_URL_COMPONENTS
  from rasa.shared.nlu.training_data.message import Message
label_desc: Use image node_id 6 for calling the WhitespaceTokenizer obj's underlying member method code with example usage: obj.tokenize(message, attribute) and returns: self