class_docstr: Base class for tokenizers.
docstr: Tokenize all training data.
docstr_tok: ["Tokenize","all","training","data","."]
code: def process_training_data(
      self, training_data: TrainingData
  ) -> TrainingData:
      """Tokenize all training data."""
      for example in training_data.training_examples:
          for attribute in MESSAGE_ATTRIBUTES:
              if (
                  example.get(attribute) is not None
                  and not example.get(attribute) == ""
              ):
                  if attribute in [
                      INTENT,
                      ACTION_NAME,
                      INTENT_RESPONSE_KEY,
                  ]:
                      tokens = self._split_name(example, attribute)
                  else:
                      tokens = self.tokenize(example, attribute)
                  example.set(TOKENS_NAMES[attribute], tokens)
      return training_data
  
imports: import abc
  import logging
  import re
  from typing import Text, List, Dict, Any, Optional
  from rasa.engine.graph import ExecutionContext, GraphComponent
  from rasa.engine.storage.resource import Resource
  from rasa.engine.storage.storage import ModelStorage
  from rasa.shared.nlu.training_data.training_data import TrainingData
  from rasa.shared.nlu.training_data.message import Message
  from rasa.nlu.constants import TOKENS_NAMES, MESSAGE_ATTRIBUTES
  from rasa.shared.nlu.constants import INTENT, INTENT_RESPONSE_KEY, RESPONSE_IDENTIFIER_DELIMITER, ACTION_NAME
  import rasa.shared.utils.io
label_desc: Use image node_id 4 for calling the Tokenizer obj's underlying member method code with example usage: obj.process_training_data(training_data) and returns: training_data