class_docstr: Featurizer using ConveRT model.
  
  Loads the ConveRT(https://github.com/PolyAI-LDN/polyai-models#convert)
  model from TFHub and computes sentence and sequence level feature representations
  for dense featurizable attributes of each message object.
class_docstr_tok: ["Featurizer","using","ConveRT","model",".","Loads","the","ConveRT","(","https",":","\/\/github.com\/PolyAI-LDN\/polyai-models","#","convert",")","model","from","TFHub","and","computes","sentence","and","sequence","level","feature","representations","for","dense","featurizable","attributes","of","each","message","object","."]
docstr: Tokenize the text using the ConveRT model.
  
  ConveRT adds a special char in front of (some) words and splits words into
  sub-words. To ensure the entity start and end values matches the token values,
  reuse the tokens that are already assigned to the message. If individual tokens
  are split up into multiple tokens, add this information to the
  respected tokens.
docstr_tok: ["Tokenize","the","text","using","the","ConveRT","model",".","ConveRT","adds","a","special","char","in","front","of","(","some",")","words","and","splits","words","into","sub-words",".","To","ensure","the","entity","start","and","end","values","matches","the","token","values",",","reuse","the","tokens","that","are","already","assigned","to","the","message",".","If","individual","tokens","are","split","up","into","multiple","tokens",",","add","this","information","to","the","respected","tokens","."]
code: def tokenize(self, message: Message, attribute: Text) -> List[Token]:
      """Tokenize the text using the ConveRT model.
  
      ConveRT adds a special char in front of (some) words and splits words into
      sub-words. To ensure the entity start and end values matches the token values,
      reuse the tokens that are already assigned to the message. If individual tokens
      are split up into multiple tokens, add this information to the
      respected tokens.
      """
      tokens_in = message.get(TOKENS_NAMES[attribute])
  
      tokens_out = []
  
      for token in tokens_in:
          # use ConveRT model to tokenize the text
          split_token_strings = self._tokenize(token.text)[0]
  
          # clean tokens (remove special chars and empty tokens)
          split_token_strings = self._clean_tokens(split_token_strings)
  
          token.set(NUMBER_OF_SUB_TOKENS, len(split_token_strings))
  
          tokens_out.append(token)
  
      message.set(TOKENS_NAMES[attribute], tokens_out)
      return tokens_out
  
imports: from __future__ import annotations
  import logging
  import os
  from typing import Any, Dict, List, Optional, Text, Tuple, Type
  import tensorflow
  from tensorflow.python.eager.wrap_function import WrappedFunction
  from tqdm import tqdm
  import numpy
  from rasa.engine.graph import GraphComponent, ExecutionContext
  from rasa.engine.recipes.default_recipe import DefaultV1Recipe
  from rasa.engine.storage.storage import ModelStorage
  from rasa.engine.storage.resource import Resource
  import rasa.shared.utils.io
  import rasa.core.utils
  from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
  from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
  from rasa.shared.nlu.training_data.training_data import TrainingData
  from rasa.shared.nlu.training_data.message import Message
  from rasa.nlu.constants import DENSE_FEATURIZABLE_ATTRIBUTES, TOKENS_NAMES, NUMBER_OF_SUB_TOKENS
  from rasa.shared.nlu.constants import TEXT, ACTION_TEXT
  from rasa.exceptions import RasaException
  import rasa.nlu.utils
  import rasa.utils.train_utils
label_desc: Use image node_id 22 for calling the ConveRTFeaturizer obj's underlying member method code with example usage: obj.tokenize(message, attribute) and returns: tokens_out