class_docstr: A featurizer that uses transformer-based language models.
  
  This component loads a pre-trained language model
  from the Transformers library (https://github.com/huggingface/transformers)
  including BERT, GPT, GPT-2, xlnet, distilbert, and roberta.
  It also tokenizes and featurizes the featurizable dense attributes of
  each message.
class_docstr_tok: ["A","featurizer","that","uses","transformer-based","language","models",".","This","component","loads","a","pre-trained","language","model","from","the","Transformers","library","(","https",":","\/\/github.com\/huggingface\/transformers",")","including","BERT",",","GPT",",","GPT-2",",","xlnet",",","distilbert",",","and","roberta",".","It","also","tokenizes","and","featurizes","the","featurizable","dense","attributes","of","each","message","."]
docstr: Tokenizes a single message example.
  
  Many language models add a special char in front of (some) words and split
  words into sub-words. To ensure the entity start and end values matches the
  token values, use the tokens produced by the Tokenizer component. If
  individual tokens are split up into multiple tokens, we add this information
  to the respected token.
  
  Args:
      message: Single message object to be processed.
      attribute: Property of message to be processed, one of ``TEXT`` or
      ``RESPONSE``.
  
  Returns: List of token strings and token ids for the corresponding
          attribute of the message.
docstr_tok: ["Tokenizes","a","single","message","example",".","Many","language","models","add","a","special","char","in","front","of","(","some",")","words","and","split","words","into","sub-words",".","To","ensure","the","entity","start","and","end","values","matches","the","token","values",",","use","the","tokens","produced","by","the","Tokenizer","component",".","If","individual","tokens","are","split","up","into","multiple","tokens",",","we","add","this","information","to","the","respected","token",".","Args",":","message",":","Single","message","object","to","be","processed",".","attribute",":","Property","of","message","to","be","processed",",","one","of","``","TEXT","``","or","``","RESPONSE","``",".","Returns",":","List","of","token","strings","and","token","ids","for","the","corresponding","attribute","of","the","message","."]
code: def _tokenize_example(
      self, message: Message, attribute: Text
  ) -> Tuple[List[Token], List[int]]:
      """Tokenizes a single message example.
  
      Many language models add a special char in front of (some) words and split
      words into sub-words. To ensure the entity start and end values matches the
      token values, use the tokens produced by the Tokenizer component. If
      individual tokens are split up into multiple tokens, we add this information
      to the respected token.
  
      Args:
          message: Single message object to be processed.
          attribute: Property of message to be processed, one of ``TEXT`` or
          ``RESPONSE``.
  
      Returns: List of token strings and token ids for the corresponding
              attribute of the message.
      """
      tokens_in = message.get(TOKENS_NAMES[attribute])
      tokens_out = []
  
      token_ids_out = []
  
      for token in tokens_in:
          # use lm specific tokenizer to further tokenize the text
          split_token_ids, split_token_strings = self._lm_tokenize(
              token.text
          )
  
          if not split_token_ids:
              # fix the situation that `token.text` only contains whitespace or other
              # special characters, which cause `split_token_ids` and
              # `split_token_strings` be empty, finally cause
              # `self._lm_specific_token_cleanup()` to raise an exception
              continue
  
          (
              split_token_ids,
              split_token_strings,
          ) = self._lm_specific_token_cleanup(
              split_token_ids, split_token_strings
          )
  
          token_ids_out += split_token_ids
  
          token.set(NUMBER_OF_SUB_TOKENS, len(split_token_strings))
  
          tokens_out.append(token)
  
      return tokens_out, token_ids_out
  
imports: from __future__ import annotations
  import numpy
  import logging
  from typing import Any, Text, List, Dict, Tuple, Type
  import tensorflow
  from rasa.engine.graph import ExecutionContext, GraphComponent
  from rasa.engine.recipes.default_recipe import DefaultV1Recipe
  from rasa.engine.storage.resource import Resource
  from rasa.engine.storage.storage import ModelStorage
  from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
  from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
  from rasa.shared.nlu.training_data.training_data import TrainingData
  from rasa.shared.nlu.training_data.message import Message
  from rasa.nlu.constants import DENSE_FEATURIZABLE_ATTRIBUTES, SEQUENCE_FEATURES, SENTENCE_FEATURES, NO_LENGTH_RESTRICTION, NUMBER_OF_SUB_TOKENS, TOKENS_NAMES
  from rasa.shared.nlu.constants import TEXT, ACTION_TEXT
  from rasa.utils import train_utils
  from rasa.utils.tensorflow.model_data import ragged_array_to_ndarray
label_desc: Use image node_id 13 for calling the LanguageModelFeaturizer obj's underlying member method code with example usage: obj._tokenize_example(message, attribute) and returns: tokens_out,  token_ids_out