class_docstr: A featurizer that uses transformer-based language models.
  
  This component loads a pre-trained language model
  from the Transformers library (https://github.com/huggingface/transformers)
  including BERT, GPT, GPT-2, xlnet, distilbert, and roberta.
  It also tokenizes and featurizes the featurizable dense attributes of
  each message.
class_docstr_tok: ["A","featurizer","that","uses","transformer-based","language","models",".","This","component","loads","a","pre-trained","language","model","from","the","Transformers","library","(","https",":","\/\/github.com\/huggingface\/transformers",")","including","BERT",",","GPT",",","GPT-2",",","xlnet",",","distilbert",",","and","roberta",".","It","also","tokenizes","and","featurizes","the","featurizable","dense","attributes","of","each","message","."]
docstr: Computes dense features of each example in the batch.
  
  We first add the special tokens corresponding to each language model. Next, we
  add appropriate padding and compute a mask for that padding so that it doesn't
  affect the feature computation. The padded batch is next fed to the language
  model and token level embeddings are computed. Using the pre-computed mask,
  embeddings for non-padding tokens are extracted and subsequently sentence
  level embeddings are computed.
  
  Args:
      batch_token_ids: List of token ids of each example in the batch.
      batch_tokens: List of token objects for each example in the batch.
      batch_examples: List of examples in the batch.
      attribute: attribute of the Message object to be processed.
      inference_mode: Whether the call is during training or during inference.
  
  Returns:
      Sentence and token level dense representations.
docstr_tok: ["Computes","dense","features","of","each","example","in","the","batch",".","We","first","add","the","special","tokens","corresponding","to","each","language","model",".","Next",",","we","add","appropriate","padding","and","compute","a","mask","for","that","padding","so","that","it","doesn't","affect","the","feature","computation",".","The","padded","batch","is","next","fed","to","the","language","model","and","token","level","embeddings","are","computed",".","Using","the","pre-computed","mask",",","embeddings","for","non-padding","tokens","are","extracted","and","subsequently","sentence","level","embeddings","are","computed",".","Args",":","batch_token_ids",":","List","of","token","ids","of","each","example","in","the","batch",".","batch_tokens",":","List","of","token","objects","for","each","example","in","the","batch",".","batch_examples",":","List","of","examples","in","the","batch",".","attribute",":","attribute","of","the","Message","object","to","be","processed",".","inference_mode",":","Whether","the","call","is","during","training","or","during","inference",".","Returns",":","Sentence","and","token","level","dense","representations","."]
code: def _get_model_features_for_batch(
      self,
      batch_token_ids: List[List[int]],
      batch_tokens: List[List[Token]],
      batch_examples: List[Message],
      attribute: Text,
      inference_mode: bool = False,
  ) -> Tuple[np.ndarray, np.ndarray]:
      """Computes dense features of each example in the batch.
  
      We first add the special tokens corresponding to each language model. Next, we
      add appropriate padding and compute a mask for that padding so that it doesn't
      affect the feature computation. The padded batch is next fed to the language
      model and token level embeddings are computed. Using the pre-computed mask,
      embeddings for non-padding tokens are extracted and subsequently sentence
      level embeddings are computed.
  
      Args:
          batch_token_ids: List of token ids of each example in the batch.
          batch_tokens: List of token objects for each example in the batch.
          batch_examples: List of examples in the batch.
          attribute: attribute of the Message object to be processed.
          inference_mode: Whether the call is during training or during inference.
  
      Returns:
          Sentence and token level dense representations.
      """
      # Let's first add tokenizer specific special tokens to all examples
      batch_token_ids_augmented = self._add_lm_specific_special_tokens(
          batch_token_ids
      )
  
      # Compute sequence lengths for all examples
      (
          actual_sequence_lengths,
          max_input_sequence_length,
      ) = self._extract_sequence_lengths(batch_token_ids_augmented)
  
      # Validate that all sequences can be processed based on their sequence
      # lengths and the maximum sequence length the model can handle
      self._validate_sequence_lengths(
          actual_sequence_lengths,
          batch_examples,
          attribute,
          inference_mode,
      )
  
      # Add padding so that whole batch can be fed to the model
      padded_token_ids = self._add_padding_to_batch(
          batch_token_ids_augmented, max_input_sequence_length
      )
  
      # Compute attention mask based on actual_sequence_length
      batch_attention_mask = self._compute_attention_mask(
          actual_sequence_lengths, max_input_sequence_length
      )
  
      # Get token level features from the model
      sequence_hidden_states = self._compute_batch_sequence_features(
          batch_attention_mask, padded_token_ids
      )
  
      # Extract features for only non-padding tokens
      sequence_nonpadded_embeddings = (
          self._extract_nonpadded_embeddings(
              sequence_hidden_states, actual_sequence_lengths
          )
      )
  
      # Extract sentence level and post-processed features
      (
          sentence_embeddings,
          sequence_embeddings,
      ) = self._post_process_sequence_embeddings(
          sequence_nonpadded_embeddings
      )
  
      # Pad zeros for examples which were truncated in inference mode.
      # This is intentionally done after sentence embeddings have been
      # extracted so that they are not affected
      sequence_embeddings = self._add_extra_padding(
          sequence_embeddings, actual_sequence_lengths
      )
  
      # shape of matrix for all sequence embeddings
      batch_dim = len(sequence_embeddings)
      seq_dim = max(e.shape[0] for e in sequence_embeddings)
      feature_dim = sequence_embeddings[0].shape[1]
      shape = (batch_dim, seq_dim, feature_dim)
  
      # align features with tokens so that we have just one vector per token
      # (don't include sub-tokens)
      sequence_embeddings = train_utils.align_token_features(
          batch_tokens, sequence_embeddings, shape
      )
  
      # sequence_embeddings is a padded numpy array
      # remove the padding, keep just the non-zero vectors
      sequence_final_embeddings = []
      for embeddings, tokens in zip(sequence_embeddings, batch_tokens):
          sequence_final_embeddings.append(embeddings[: len(tokens)])
  
      return sentence_embeddings, ragged_array_to_ndarray(
          sequence_final_embeddings
      )
  
imports: from __future__ import annotations
  import numpy
  import logging
  from typing import Any, Text, List, Dict, Tuple, Type
  import tensorflow
  from rasa.engine.graph import ExecutionContext, GraphComponent
  from rasa.engine.recipes.default_recipe import DefaultV1Recipe
  from rasa.engine.storage.resource import Resource
  from rasa.engine.storage.storage import ModelStorage
  from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
  from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
  from rasa.shared.nlu.training_data.training_data import TrainingData
  from rasa.shared.nlu.training_data.message import Message
  from rasa.nlu.constants import DENSE_FEATURIZABLE_ATTRIBUTES, SEQUENCE_FEATURES, SENTENCE_FEATURES, NO_LENGTH_RESTRICTION, NUMBER_OF_SUB_TOKENS, TOKENS_NAMES
  from rasa.shared.nlu.constants import TEXT, ACTION_TEXT
  from rasa.utils import train_utils
  from rasa.utils.tensorflow.model_data import ragged_array_to_ndarray
label_desc: Use image node_id 22 for calling the LanguageModelFeaturizer obj's underlying member method code with example usage: obj._get_model_features_for_batch(batch_token_ids, batch_tokens, batch_examples, attribute, inference_mode) and returns: sentence_embeddings,  ragged_array_to_ndarray