class_docstr: A featurizer that uses transformer-based language models.
  
  This component loads a pre-trained language model
  from the Transformers library (https://github.com/huggingface/transformers)
  including BERT, GPT, GPT-2, xlnet, distilbert, and roberta.
  It also tokenizes and featurizes the featurizable dense attributes of
  each message.
class_docstr_tok: ["A","featurizer","that","uses","transformer-based","language","models",".","This","component","loads","a","pre-trained","language","model","from","the","Transformers","library","(","https",":","\/\/github.com\/huggingface\/transformers",")","including","BERT",",","GPT",",","GPT-2",",","xlnet",",","distilbert",",","and","roberta",".","It","also","tokenizes","and","featurizes","the","featurizable","dense","attributes","of","each","message","."]
docstr: Extracts the sequence length for each example and maximum sequence length.
  
  Args:
      batch_token_ids: List of token ids for each example in the batch.
  
  Returns:
      Tuple consisting of: the actual sequence lengths for each example,
      and the maximum input sequence length (taking into account the
      maximum sequence length that the model can handle.
docstr_tok: ["Extracts","the","sequence","length","for","each","example","and","maximum","sequence","length",".","Args",":","batch_token_ids",":","List","of","token","ids","for","each","example","in","the","batch",".","Returns",":","Tuple","consisting","of",":","the","actual","sequence","lengths","for","each","example",",","and","the","maximum","input","sequence","length","(","taking","into","account","the","maximum","sequence","length","that","the","model","can","handle","."]
code: def _extract_sequence_lengths(
      self, batch_token_ids: List[List[int]]
  ) -> Tuple[List[int], int]:
      """Extracts the sequence length for each example and maximum sequence length.
  
      Args:
          batch_token_ids: List of token ids for each example in the batch.
  
      Returns:
          Tuple consisting of: the actual sequence lengths for each example,
          and the maximum input sequence length (taking into account the
          maximum sequence length that the model can handle.
      """
      # Compute max length across examples
      max_input_sequence_length = 0
      actual_sequence_lengths = []
  
      for example_token_ids in batch_token_ids:
          sequence_length = len(example_token_ids)
          actual_sequence_lengths.append(sequence_length)
          max_input_sequence_length = max(
              max_input_sequence_length, len(example_token_ids)
          )
  
      # Take into account the maximum sequence length the model can handle
      max_input_sequence_length = (
          max_input_sequence_length
          if self.max_model_sequence_length == NO_LENGTH_RESTRICTION
          else min(
              max_input_sequence_length, self.max_model_sequence_length
          )
      )
  
      return actual_sequence_lengths, max_input_sequence_length
  
imports: from __future__ import annotations
  import numpy
  import logging
  from typing import Any, Text, List, Dict, Tuple, Type
  import tensorflow
  from rasa.engine.graph import ExecutionContext, GraphComponent
  from rasa.engine.recipes.default_recipe import DefaultV1Recipe
  from rasa.engine.storage.resource import Resource
  from rasa.engine.storage.storage import ModelStorage
  from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer
  from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
  from rasa.shared.nlu.training_data.training_data import TrainingData
  from rasa.shared.nlu.training_data.message import Message
  from rasa.nlu.constants import DENSE_FEATURIZABLE_ATTRIBUTES, SEQUENCE_FEATURES, SENTENCE_FEATURES, NO_LENGTH_RESTRICTION, NUMBER_OF_SUB_TOKENS, TOKENS_NAMES
  from rasa.shared.nlu.constants import TEXT, ACTION_TEXT
  from rasa.utils import train_utils
  from rasa.utils.tensorflow.model_data import ragged_array_to_ndarray
label_desc: Use image node_id 16 for calling the LanguageModelFeaturizer obj's underlying member method code with example usage: obj._extract_sequence_lengths(batch_token_ids) and returns: actual_sequence_lengths,  max_input_sequence_length