text: import typing
  from typing import Any
  
  from rasa.nlu.components import Component
  from rasa.nlu.config import RasaNLUModelConfig
  from rasa.nlu.tokenizers import Token, Tokenizer
  from rasa.nlu.training_data import Message, TrainingData
  
  from rasa.nlu.constants import (
      MESSAGE_RESPONSE_ATTRIBUTE,
      MESSAGE_INTENT_ATTRIBUTE,
      MESSAGE_TEXT_ATTRIBUTE,
      MESSAGE_TOKENS_NAMES,
      MESSAGE_ATTRIBUTES,
      MESSAGE_SPACY_FEATURES_NAMES,
      MESSAGE_VECTOR_FEATURE_NAMES,
      SPACY_FEATURIZABLE_ATTRIBUTES,
  )
  
  if typing.TYPE_CHECKING:
      from spacy.tokens.doc import Doc  # pytype: disable=import-error
  
  
  class SpacyTokenizer(Tokenizer, Component):
  
      provides = [
          MESSAGE_TOKENS_NAMES[attribute] for attribute in SPACY_FEATURIZABLE_ATTRIBUTES
      ]
  
      requires = [
          MESSAGE_SPACY_FEATURES_NAMES[attribute]
          for attribute in SPACY_FEATURIZABLE_ATTRIBUTES
      ]
  
      def train(
          self, training_data: TrainingData, config: RasaNLUModelConfig, **kwargs: Any
      ) -> None:
  
          for example in training_data.training_examples:
  
              for attribute in SPACY_FEATURIZABLE_ATTRIBUTES:
  
                  attribute_doc = self.get_doc(example, attribute)
  
                  if attribute_doc is not None:
                      example.set(
                          MESSAGE_TOKENS_NAMES[attribute], self.tokenize(attribute_doc)
                      )
  
      def get_doc(self, message, attribute) -> Any:
  
          return message.get(MESSAGE_SPACY_FEATURES_NAMES[attribute])
  
      def process(self, message: Message, **kwargs: Any) -> None:
  
          message.set(
              MESSAGE_TOKENS_NAMES[MESSAGE_TEXT_ATTRIBUTE],
              self.tokenize(self.get_doc(message, MESSAGE_TEXT_ATTRIBUTE)),
          )
  
      def tokenize(self, doc: "Doc") -> typing.List[Token]:
  
          return [Token(t.text, t.idx) for t in doc]
  