class_docstr: Tokenizes messages using the `mitie` library..
class_docstr_tok: ["Tokenizes","messages","using","the","`","mitie","`","library",".."]
docstr: Tokenizes the text of the provided attribute of the incoming message.
docstr_tok: ["Tokenizes","the","text","of","the","provided","attribute","of","the","incoming","message","."]
code: def tokenize(self, message: Message, attribute: Text) -> List[Token]:
      """Tokenizes the text of the provided attribute of the incoming message."""
      import mitie
  
      text = message.get(attribute)
  
      encoded_sentence = text.encode(DEFAULT_ENCODING)
      tokenized = mitie.tokenize_with_offsets(encoded_sentence)
      tokens = [
          self._token_from_offset(token, offset, encoded_sentence)
          for token, offset in tokenized
      ]
  
      return self._apply_token_pattern(tokens)
  
file_path: rasa/rasa/nlu/tokenizers/mitie_tokenizer.py
imports: from __future__ import annotations
  from typing import List, Text, Dict, Any
  from rasa.engine.graph import ExecutionContext
  from rasa.engine.recipes.default_recipe import DefaultV1Recipe
  from rasa.engine.storage.resource import Resource
  from rasa.engine.storage.storage import ModelStorage
  from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
  from rasa.shared.nlu.training_data.message import Message
  from rasa.shared.utils.io import DEFAULT_ENCODING
label_desc: Use image node_id 4 for calling the MitieTokenizer obj's underlying member method code with example usage: obj.tokenize(message, attribute) and returns: self