class_docstr: Transformer language model.
  
  Arguments:
      transformer_hparams: transformer hyperparameters
      vocab_size: vocabulary size
      max_sequence_length: maximum size of sequence. This
                           is used for positional embedding
      embedding_dropout_prob: dropout probability for embeddings
      num_tokentypes: size of the token-type embeddings. 0 value
                      will ignore this embedding
code: def __init__(
      self,
      config: ModelParallelConfig,
      init_method,
      output_layer_init_method,
      encoder_attn_mask_type,
      vocab_size,
      max_position_embeddings,
      hidden_size,
      ffn_hidden_size,
      num_layers,
      num_tokentypes,
      num_attention_heads,
      apply_query_key_layer_scaling=True,
      kv_channels=None,
      add_decoder=False,
      decoder_attn_mask_type=AttnMaskType.causal,
      add_pooler=False,
      pre_process=True,
      post_process=True,
      hidden_dropout=0.1,
      attention_dropout=0.1,
      ffn_dropout=0.0,
      precision=16,
      fp32_residual_connection=False,
      activations_checkpoint_method=None,
      activations_checkpoint_num_layers=1,
      normalization="layernorm",
      layernorm_epsilon=1e-5,
      bias_activation_fusion=True,
      bias_dropout_add_fusion=True,
      bias=True,
      masked_softmax_fusion=True,
      activation="gelu",
      headscale=False,
      transformer_block_type="pre_ln",
      normalize_attention_scores=True,
      position_embedding_type="learned_absolute",
      rotary_percentage=1.0,
      multi_query_attention=False,
      share_embeddings_and_output_weights=True,
      persist_layer_norm=False,
      openai_gelu=False,
      onnx_safe=False,
      megatron_legacy=False,
      activations_checkpoint_granularity=None,
      activations_checkpoint_layers_per_pipeline=None,
      transformer_engine=False,
      fp8=False,
      fp8_e4m3=False,
      fp8_hybrid=False,
      fp8_margin=0,
      fp8_interval=1,
      fp8_amax_history_len=1024,
      fp8_amax_compute_algo="max",
      reduce_amax=True,
      use_emha=False,
      ub_tp_comm_overlap=False,
      use_flash_attention=False,
      seq_len_interpolation_factor=None,
      rotary_base=10000,
  ):
      super(TransformerLanguageModel, self).__init__(
          config=config,
          share_token_embeddings=share_embeddings_and_output_weights,
      )
  
      self.pre_process = pre_process
      self.post_process = post_process
      self.hidden_size = hidden_size
      self.num_layers = num_layers
      self.vocab_size = vocab_size
      self.max_position_embeddings = max_position_embeddings
      self.num_tokentypes = num_tokentypes
      self.init_method = init_method
      self.encoder_attn_mask_type = encoder_attn_mask_type
      self.add_decoder = add_decoder
      self.decoder_attn_mask_type = decoder_attn_mask_type
      self.add_pooler = add_pooler
      self.hidden_dropout = hidden_dropout
      self.output_layer_init_method = output_layer_init_method
      self.position_embedding_type = position_embedding_type
      self.share_embeddings_and_output_weights = (
          share_embeddings_and_output_weights
      )
      self.sequence_parallel = config.sequence_parallel
      self.context_parallel = (
          parallel_state.get_context_parallel_world_size() > 1
      )
      if kv_channels is None:
          assert (
              hidden_size % num_attention_heads == 0
          ), "hidden_size must be divisible by num_attention_heads if kv_channels is None"
          kv_channels = hidden_size // num_attention_heads
  
      # Embeddings.
      if self.pre_process:
          self.embedding = Embedding(
              config=config,
              hidden_size=self.hidden_size,
              vocab_size=self.vocab_size,
              max_sequence_length=self.max_position_embeddings,
              init_method=self.init_method,
              num_tokentypes=self.num_tokentypes,
              embedding_dropout_prob=self.hidden_dropout,
              position_embedding_type=position_embedding_type,
              fp32_residual_connection=fp32_residual_connection,
          )
          self._embedding_key = "embedding"
  
      if position_embedding_type == "rope":
          rotary_dim = (
              self.hidden_size // num_attention_heads
              if kv_channels is None
              else kv_channels
          )
          assert 0 < rotary_percentage <= 1
          if rotary_percentage < 1:
              rotary_dim = int(rotary_dim * rotary_percentage)
          self.rotary_pos_emb = RotaryEmbedding(
              rotary_dim,
              seq_len_interpolation_factor=seq_len_interpolation_factor,
              pretrained_max_position_embeddings=max_position_embeddings,
              rotary_base=rotary_base,
          )
  
      elif position_embedding_type == "alibi":
          # TODO: If this is used for encoder-decodemax_position_embeddingsr model, implement proper logic and following
          # addition for decoder. Currently it is only used for decoder model only.
          # Encoder-decoder model, such as T5 is implemented in token_level_encoder_decoder.py
          self.encoder_relative_position_embedding = (
              ALiBiRelativePositionEmbedding(
                  bidirectional=encoder_attn_mask_type
                  != AttnMaskType.causal,
                  num_attention_heads=num_attention_heads,
                  layer_type=LayerType.encoder,
                  num_attention_heads_alibi=None,
                  max_seq_len=max_position_embeddings,
              )
          )
  
      elif position_embedding_type == "kerple":
          # TODO: If this is used for encoder-decodemax_position_embeddingsr model, implement proper logic and following
          # addition for decoder. Currently it is only used for decoder model only.
          # Encoder-decoder model, such as T5 is implemented in token_level_encoder_decoder.py
          self.encoder_relative_position_embedding = (
              KERPLERelativePositionEmbedding(
                  bidirectional=encoder_attn_mask_type
                  != AttnMaskType.causal,
                  num_attention_heads=num_attention_heads,
                  layer_type=LayerType.encoder,
                  num_attention_heads_kerple=None,
                  max_seq_len=max_position_embeddings,
              )
          )
          assert (
              use_flash_attention == False
          )  # flash-attention not supported with kerple at this point
  
      elif position_embedding_type == "sandwich":
          self.encoder_relative_position_embedding = (
              SandwichRelativePositionEmbedding(
                  bidirectional=encoder_attn_mask_type
                  != AttnMaskType.causal,
                  num_attention_heads=num_attention_heads,
                  layer_type=LayerType.encoder,
                  hidden_size=self.hidden_size // num_attention_heads
                  if kv_channels is None
                  else kv_channels,
                  max_seq_len=max_position_embeddings,
              )
          )
  
      # Transformer.
      self.encoder = ParallelTransformer(
          config=config,
          init_method=self.init_method,
          output_layer_init_method=self.output_layer_init_method,
          num_layers=self.num_layers,
          hidden_size=self.hidden_size,
          num_attention_heads=num_attention_heads,
          apply_query_key_layer_scaling=apply_query_key_layer_scaling,
          kv_channels=kv_channels,
          ffn_hidden_size=ffn_hidden_size,
          self_attn_mask_type=self.encoder_attn_mask_type,
          pre_process=self.pre_process,
          post_process=self.post_process,
          precision=precision,
          fp32_residual_connection=fp32_residual_connection,
          activations_checkpoint_method=activations_checkpoint_method,
          activations_checkpoint_num_layers=activations_checkpoint_num_layers,
          normalization=normalization,
          layernorm_epsilon=layernorm_epsilon,
          hidden_dropout=hidden_dropout,
          attention_dropout=attention_dropout,
          ffn_dropout=ffn_dropout,
          persist_layer_norm=persist_layer_norm,
          openai_gelu=openai_gelu,
          onnx_safe=onnx_safe,
          bias=bias,
          bias_activation_fusion=bias_activation_fusion,
          bias_dropout_add_fusion=bias_dropout_add_fusion,
          masked_softmax_fusion=masked_softmax_fusion,
          activation=activation,
          headscale=headscale,
          transformer_block_type=transformer_block_type,
          normalize_attention_scores=normalize_attention_scores,
          multi_query_attention=multi_query_attention,
          megatron_legacy=megatron_legacy,
          activations_checkpoint_granularity=activations_checkpoint_granularity,
          activations_checkpoint_layers_per_pipeline=activations_checkpoint_layers_per_pipeline,
          transformer_engine=transformer_engine,
          fp8=fp8,
          fp8_e4m3=fp8_e4m3,
          fp8_hybrid=fp8_hybrid,
          fp8_margin=fp8_margin,
          fp8_interval=fp8_interval,
          fp8_amax_history_len=fp8_amax_history_len,
          fp8_amax_compute_algo=fp8_amax_compute_algo,
          reduce_amax=reduce_amax,
          use_emha=use_emha,
          ub_tp_comm_overlap=ub_tp_comm_overlap,
          position_embedding_type=position_embedding_type,
          use_flash_attention=use_flash_attention,
      )
      self._encoder_key = "encoder"
  
      # Decoder
      if self.add_decoder:
          self.decoder = ParallelTransformer(
              config=config,
              layer_type=LayerType.decoder,
              self_attn_mask_type=self.decoder_attn_mask_type,
              init_method=self.init_method,
              output_layer_init_method=self.output_layer_init_method,
              num_layers=self.num_layers,
              hidden_size=self.hidden_size,
              num_attention_heads=num_attention_heads,
              apply_query_key_layer_scaling=apply_query_key_layer_scaling,
              kv_channels=kv_channels,
              ffn_hidden_size=ffn_hidden_size,
              pre_process=self.pre_process,
              post_process=self.post_process,
              precision=precision,
              fp32_residual_connection=fp32_residual_connection,
              activations_checkpoint_method=activations_checkpoint_method,
              activations_checkpoint_num_layers=activations_checkpoint_num_layers,
              normalization=normalization,
              layernorm_epsilon=layernorm_epsilon,
              hidden_dropout=hidden_dropout,
              attention_dropout=attention_dropout,
              bias_activation_fusion=bias_activation_fusion,
              bias_dropout_add_fusion=bias_dropout_add_fusion,
              masked_softmax_fusion=masked_softmax_fusion,
              persist_layer_norm=persist_layer_norm,
              openai_gelu=openai_gelu,
              onnx_safe=onnx_safe,
              megatron_legacy=megatron_legacy,
              activations_checkpoint_granularity=activations_checkpoint_granularity,
              activations_checkpoint_layers_per_pipeline=activations_checkpoint_layers_per_pipeline,
              transformer_engine=transformer_engine,
              position_embedding_type=position_embedding_type,
              use_flash_attention=use_flash_attention,
          )
          self._decoder_key = "decoder"
  
      if self.post_process:
          # Pooler.
          if self.add_pooler:
              self.pooler = Pooler(
                  self.hidden_size,
                  self.init_method,
                  sequence_parallel=self.sequence_parallel,
              )
              self._pooler_key = "pooler"
  
          if not self.share_embeddings_and_output_weights:
              self.output_layer = tensor_parallel.ColumnParallelLinear(
                  self.hidden_size,
                  self.vocab_size,
                  config=config,
                  bias=False,  # Setting bias to False always to keep it consistent with embedding tying that also does not have a bias.
                  init_method=self.init_method,
              )
              self._output_layer_key = "output_layer"
      self.set_accepted_adapter_types(
          [PromptEncoderAdapterConfig._target_]
      )
  
imports: from ast import Mod
  import torch
  from nemo.collections.nlp.modules.common.megatron.adapters.parallel_adapters import AdapterName, PromptEncoderAdapterConfig
  from nemo.collections.nlp.modules.common.megatron.layer_type import LayerType
  from nemo.collections.nlp.modules.common.megatron.module import MegatronModule
  from nemo.collections.nlp.modules.common.megatron.position_embedding import ALiBiRelativePositionEmbedding, KERPLERelativePositionEmbedding, RotaryEmbedding, SandwichRelativePositionEmbedding
  from nemo.collections.nlp.modules.common.megatron.transformer import ParallelTransformer
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults, get_linear_layer, init_method_normal, scaled_init_method_normal
  from nemo.collections.nlp.parts import utils_funcs
  from nemo.core import adapter_mixins
label_desc: Use image node_id 1 to create a new TransformerLanguageModel object from inherited base classes: MegatronModule, adapter_mixins with example: obj = TransformerLanguageModel(config, init_method, output_layer_init_method, encoder_attn_mask_type, vocab_size, max_position_embeddings, hidden_size, ffn_hidden_size, num_layers, num_tokentypes, num_attention_heads, apply_query_key_layer_scaling, kv_channels, add_decoder, decoder_attn_mask_type, add_pooler, pre_process, post_process, hidden_dropout, attention_dropout, ffn_dropout, precision, fp32_residual_connection, activations_checkpoint_method, activations_checkpoint_num_layers, normalization, layernorm_epsilon, bias_activation_fusion, bias_dropout_add_fusion, bias, masked_softmax_fusion, activation, headscale, transformer_block_type, normalize_attention_scores, position_embedding_type, rotary_percentage, multi_query_attention, share_embeddings_and_output_weights, persist_layer_norm, openai_gelu, onnx_safe, megatron_legacy, activations_checkpoint_granularity, activations_checkpoint_layers_per_pipeline, transformer_engine, fp8, fp8_e4m3, fp8_hybrid, fp8_margin, fp8_interval, fp8_amax_history_len, fp8_amax_compute_algo, reduce_amax, use_emha, ub_tp_comm_overlap, use_flash_attention, seq_len_interpolation_factor, rotary_base)