class_docstr: A featurizer that uses transformer-based language models.
  
  This component loads a pre-trained language model
  from the Transformers library (https://github.com/huggingface/transformers)
  including BERT, GPT, GPT-2, xlnet, distilbert, and roberta.
  It also tokenizes and featurizes the featurizable dense attributes of
  each message.
class_docstr_tok: ["A","featurizer","that","uses","transformer-based","language","models",".","This","component","loads","a","pre-trained","language","model","from","the","Transformers","library","(","https",":","\/\/github.com\/huggingface\/transformers",")","including","BERT",",","GPT",",","GPT-2",",","xlnet",",","distilbert",",","and","roberta",".","It","also","tokenizes","and","featurizes","the","featurizable","dense","attributes","of","each","message","."]
docstr: Cleans up special chars added by tokenizers of language models.
  
  Many language models add a special char in front/back of (some) words. We clean
  up those chars as they are not
  needed once the features are already computed.
  
  Args:
      split_token_ids: List of token ids received as output from the language
      model specific tokenizer.
      token_strings: List of token strings received as output from the language
      model specific tokenizer.
  
  Returns: Cleaned up token ids and token strings.
docstr_tok: ["Cleans","up","special","chars","added","by","tokenizers","of","language","models",".","Many","language","models","add","a","special","char","in","front\/back","of","(","some",")","words",".","We","clean","up","those","chars","as","they","are","not","needed","once","the","features","are","already","computed",".","Args",":","split_token_ids",":","List","of","token","ids","received","as","output","from","the","language","model","specific","tokenizer",".","token_strings",":","List","of","token","strings","received","as","output","from","the","language","model","specific","tokenizer",".","Returns",":","Cleaned","up","token","ids","and","token","strings","."]
code: def _lm_specific_token_cleanup(
      self, split_token_ids: List[int], token_strings: List[Text]
  ) -> Tuple[List[int], List[Text]]:
      """Cleans up special chars added by tokenizers of language models.
  
      Many language models add a special char in front/back of (some) words. We clean
      up those chars as they are not
      needed once the features are already computed.
  
      Args:
          split_token_ids: List of token ids received as output from the language
          model specific tokenizer.
          token_strings: List of token strings received as output from the language
          model specific tokenizer.
  
      Returns: Cleaned up token ids and token strings.
      """
      from rasa.nlu.utils.hugging_face.registry import (
          model_tokens_cleaners,
      )
  
      return model_tokens_cleaners[self.model_name](
          split_token_ids, token_strings
      )
  
imports: from __future__ import annotations
  import numpy
  import logging
  from typing import Any, Text, List, Dict, Tuple, Type
  import tensorflow
  from rasa.engine.graph import ExecutionContext, GraphComponent
  from rasa.engine.recipes.default_recipe import DefaultV1Recipe
  from rasa.engine.storage.resource import Resource
  from rasa.engine.storage.storage import ModelStorage
  from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
  from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
  from rasa.shared.nlu.training_data.training_data import TrainingData
  from rasa.shared.nlu.training_data.message import Message
  from rasa.nlu.constants import DENSE_FEATURIZABLE_ATTRIBUTES, SEQUENCE_FEATURES, SENTENCE_FEATURES, NO_LENGTH_RESTRICTION, NUMBER_OF_SUB_TOKENS, TOKENS_NAMES
  from rasa.shared.nlu.constants import TEXT, ACTION_TEXT
  from rasa.utils import train_utils
  from rasa.utils.tensorflow.model_data import ragged_array_to_ndarray
label_desc: Use image node_id 11 for calling the LanguageModelFeaturizer obj's underlying member method code with example usage: obj._lm_specific_token_cleanup(split_token_ids, token_strings) and returns: model_tokens_cleaners