class_docstr: Base class for tokenizers.
docstr: Tokenize the incoming messages.
docstr_tok: ["Tokenize","the","incoming","messages","."]
code: def process(self, messages: List[Message]) -> List[Message]:
      """Tokenize the incoming messages."""
      for message in messages:
          for attribute in MESSAGE_ATTRIBUTES:
              if isinstance(message.get(attribute), str):
                  if attribute in [
                      INTENT,
                      ACTION_NAME,
                      RESPONSE_IDENTIFIER_DELIMITER,
                  ]:
                      tokens = self._split_name(message, attribute)
                  else:
                      tokens = self.tokenize(message, attribute)
  
                  message.set(TOKENS_NAMES[attribute], tokens)
      return messages
  
imports: import abc
  import logging
  import re
  from typing import Text, List, Dict, Any, Optional
  from rasa.engine.graph import ExecutionContext, GraphComponent
  from rasa.engine.storage.resource import Resource
  from rasa.engine.storage.storage import ModelStorage
  from rasa.shared.nlu.training_data.training_data import TrainingData
  from rasa.shared.nlu.training_data.message import Message
  from rasa.nlu.constants import TOKENS_NAMES, MESSAGE_ATTRIBUTES
  from rasa.shared.nlu.constants import INTENT, INTENT_RESPONSE_KEY, RESPONSE_IDENTIFIER_DELIMITER, ACTION_NAME
  import rasa.shared.utils.io
label_desc: Use image node_id 5 for calling the Tokenizer obj's underlying member method code with example usage: obj.process(messages) and returns: messages