class_docstr: A featurizer that uses transformer-based language models.
  
  This component loads a pre-trained language model
  from the Transformers library (https://github.com/huggingface/transformers)
  including BERT, GPT, GPT-2, xlnet, distilbert, and roberta.
  It also tokenizes and featurizes the featurizable dense attributes of
  each message.
class_docstr_tok: ["A","featurizer","that","uses","transformer-based","language","models",".","This","component","loads","a","pre-trained","language","model","from","the","Transformers","library","(","https",":","\/\/github.com\/huggingface\/transformers",")","including","BERT",",","GPT",",","GPT-2",",","xlnet",",","distilbert",",","and","roberta",".","It","also","tokenizes","and","featurizes","the","featurizable","dense","attributes","of","each","message","."]
docstr: Computes tokens and dense features for each message in training data.
  
  Args:
      training_data: NLU training data to be tokenized and featurized
      config: NLU pipeline config consisting of all components.
docstr_tok: ["Computes","tokens","and","dense","features","for","each","message","in","training","data",".","Args",":","training_data",":","NLU","training","data","to","be","tokenized","and","featurized","config",":","NLU","pipeline","config","consisting","of","all","components","."]
code: def process_training_data(
      self, training_data: TrainingData
  ) -> TrainingData:
      """Computes tokens and dense features for each message in training data.
  
      Args:
          training_data: NLU training data to be tokenized and featurized
          config: NLU pipeline config consisting of all components.
      """
      batch_size = 64
  
      for attribute in DENSE_FEATURIZABLE_ATTRIBUTES:
          non_empty_examples = list(
              filter(
                  lambda x: x.get(attribute),
                  training_data.training_examples,
              )
          )
  
          batch_start_index = 0
  
          while batch_start_index < len(non_empty_examples):
              batch_end_index = min(
                  batch_start_index + batch_size,
                  len(non_empty_examples),
              )
              # Collect batch examples
              batch_messages = non_empty_examples[
                  batch_start_index:batch_end_index
              ]
  
              # Construct a doc with relevant features
              # extracted(tokens, dense_features)
              batch_docs = self._get_docs_for_batch(
                  batch_messages, attribute
              )
  
              for index, ex in enumerate(batch_messages):
                  self._set_lm_features(
                      batch_docs[index], ex, attribute
                  )
              batch_start_index += batch_size
  
      return training_data
  
imports: from __future__ import annotations
  import numpy
  import logging
  from typing import Any, Text, List, Dict, Tuple, Type
  import tensorflow
  from rasa.engine.graph import ExecutionContext, GraphComponent
  from rasa.engine.recipes.default_recipe import DefaultV1Recipe
  from rasa.engine.storage.resource import Resource
  from rasa.engine.storage.storage import ModelStorage
  from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
  from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
  from rasa.shared.nlu.training_data.training_data import TrainingData
  from rasa.shared.nlu.training_data.message import Message
  from rasa.nlu.constants import DENSE_FEATURIZABLE_ATTRIBUTES, SEQUENCE_FEATURES, SENTENCE_FEATURES, NO_LENGTH_RESTRICTION, NUMBER_OF_SUB_TOKENS, TOKENS_NAMES
  from rasa.shared.nlu.constants import TEXT, ACTION_TEXT
  from rasa.utils import train_utils
  from rasa.utils.tensorflow.model_data import ragged_array_to_ndarray
label_desc: Use image node_id 24 for calling the LanguageModelFeaturizer obj's underlying member method code with example usage: obj.process_training_data(training_data) and returns: training_data