class_docstr: GPT-2 Language model.
class_docstr_tok: ["GPT-2","Language","model","."]
code: def __init__(
      self,
      config: ModelParallelConfig,
      vocab_size,
      hidden_size,
      max_position_embeddings,
      num_layers,
      num_attention_heads,
      ffn_hidden_size,
      apply_query_key_layer_scaling=False,
      kv_channels=None,
      num_tokentypes=0,
      parallel_output=True,
      pre_process=True,
      post_process=True,
      init_method_std=0.02,
      use_scaled_init_method=True,
      fp16_lm_cross_entropy=False,
      hidden_dropout=0.1,
      attention_dropout=0.1,
      ffn_dropout=0.0,
      precision=16,
      fp32_residual_connection=False,
      activations_checkpoint_granularity=None,
      activations_checkpoint_method=None,
      activations_checkpoint_num_layers=1,
      activations_checkpoint_layers_per_pipeline=None,
      normalization="layernorm",
      layernorm_epsilon=1e-5,
      bias=True,
      bias_activation_fusion=True,
      bias_dropout_add_fusion=True,
      masked_softmax_fusion=True,
      activation="gelu",
      headscale=False,
      transformer_block_type="pre_ln",
      normalize_attention_scores=True,
      position_embedding_type="learned_absolute",
      rotary_percentage=1.0,
      attention_type="multihead",
      share_embeddings_and_output_weights=True,
      persist_layer_norm=False,
      openai_gelu=False,
      megatron_legacy=False,
      onnx_safe=False,
      transformer_engine=False,
      fp8=False,
      fp8_e4m3=False,
      fp8_hybrid=False,
      fp8_margin=0,
      fp8_interval=1,
      fp8_amax_history_len=1024,
      fp8_amax_compute_algo="max",
      reduce_amax=True,
      use_emha=False,
      ub_tp_comm_overlap=False,
      use_flash_attention=False,
      seq_len_interpolation_factor=None,
      rotary_base=10000,
  ):
      super(GPTModel, self).__init__(
          config=config,
          share_token_embeddings=share_embeddings_and_output_weights,
      )
  
      self.parallel_output = parallel_output
      self.pre_process = pre_process
      self.post_process = post_process
      self.fp16_lm_cross_entropy = fp16_lm_cross_entropy
      self.sequence_parallel = self.config.sequence_parallel
      self.share_embeddings_and_output_weights = (
          share_embeddings_and_output_weights
      )
  
      if kv_channels is None:
          assert (
              hidden_size % num_attention_heads == 0
          ), "hidden_size must be divisible by num_attention_heads if kv_channels is None"
          kv_channels = hidden_size // num_attention_heads
  
      scaled_init_method = (
          scaled_init_method_normal(init_method_std, num_layers)
          if use_scaled_init_method
          else init_method_normal(init_method_std)
      )
      (
          self.language_model,
          self._language_model_key,
      ) = get_language_model(
          config=config,
          vocab_size=vocab_size,
          hidden_size=hidden_size,
          hidden_dropout=hidden_dropout,
          attention_dropout=attention_dropout,
          ffn_dropout=ffn_dropout,
          num_tokentypes=num_tokentypes,
          max_position_embeddings=max_position_embeddings,
          num_layers=num_layers,
          num_attention_heads=num_attention_heads,
          apply_query_key_layer_scaling=apply_query_key_layer_scaling,
          kv_channels=kv_channels,
          ffn_hidden_size=ffn_hidden_size,
          add_pooler=False,
          encoder_attn_mask_type=AttnMaskType.causal,
          init_method=init_method_normal(init_method_std),
          scaled_init_method=scaled_init_method,
          pre_process=self.pre_process,
          post_process=self.post_process,
          init_method_std=init_method_std,
          precision=precision,
          fp32_residual_connection=fp32_residual_connection,
          activations_checkpoint_granularity=activations_checkpoint_granularity,
          activations_checkpoint_method=activations_checkpoint_method,
          activations_checkpoint_num_layers=activations_checkpoint_num_layers,
          activations_checkpoint_layers_per_pipeline=activations_checkpoint_layers_per_pipeline,
          normalization=normalization,
          layernorm_epsilon=layernorm_epsilon,
          rotary_percentage=rotary_percentage,
          share_embeddings_and_output_weights=share_embeddings_and_output_weights,
          bias=bias,
          bias_activation_fusion=bias_activation_fusion,
          bias_dropout_add_fusion=bias_dropout_add_fusion,
          masked_softmax_fusion=masked_softmax_fusion,
          activation=activation,
          headscale=headscale,
          transformer_block_type=transformer_block_type,
          normalize_attention_scores=normalize_attention_scores,
          position_embedding_type=position_embedding_type,
          attention_type=attention_type,
          persist_layer_norm=persist_layer_norm,
          openai_gelu=openai_gelu,
          onnx_safe=onnx_safe,
          megatron_legacy=megatron_legacy,
          transformer_engine=transformer_engine,
          fp8=fp8,
          fp8_e4m3=fp8_e4m3,
          fp8_hybrid=fp8_hybrid,
          fp8_margin=fp8_margin,
          fp8_interval=fp8_interval,
          fp8_amax_history_len=fp8_amax_history_len,
          fp8_amax_compute_algo=fp8_amax_compute_algo,
          reduce_amax=reduce_amax,
          use_emha=use_emha,
          ub_tp_comm_overlap=ub_tp_comm_overlap,
          use_flash_attention=use_flash_attention,
          seq_len_interpolation_factor=seq_len_interpolation_factor,
          rotary_base=rotary_base,
      )
  
      if self.share_embeddings_and_output_weights:
          self.initialize_word_embeddings(
              init_method=init_method_normal(init_method_std),
              vocab_size=vocab_size,
              hidden_size=hidden_size,
          )
  
file_path: NeMo/nemo/collections/nlp/models/language_modeling/megatron/gpt_model.py
imports: import torch
  from nemo.collections.nlp.modules.common.megatron.language_model import get_language_model
  from nemo.collections.nlp.modules.common.megatron.module import MegatronModule
  from nemo.collections.nlp.modules.common.megatron.utils import ApexGuardDefaults, init_method_normal, parallel_lm_logits, scaled_init_method_normal
label_desc: Use image node_id 1 to create a new GPTModel object from inherited base classes: MegatronModule with example: obj = GPTModel(config, vocab_size, hidden_size, max_position_embeddings, num_layers, num_attention_heads, ffn_hidden_size, apply_query_key_layer_scaling, kv_channels, num_tokentypes, parallel_output, pre_process, post_process, init_method_std, use_scaled_init_method, fp16_lm_cross_entropy, hidden_dropout, attention_dropout, ffn_dropout, precision, fp32_residual_connection, activations_checkpoint_granularity, activations_checkpoint_method, activations_checkpoint_num_layers, activations_checkpoint_layers_per_pipeline, normalization, layernorm_epsilon, bias, bias_activation_fusion, bias_dropout_add_fusion, masked_softmax_fusion, activation, headscale, transformer_block_type, normalize_attention_scores, position_embedding_type, rotary_percentage, attention_type, share_embeddings_and_output_weights, persist_layer_norm, openai_gelu, megatron_legacy, onnx_safe, transformer_engine, fp8, fp8_e4m3, fp8_hybrid, fp8_margin, fp8_interval, fp8_amax_history_len, fp8_amax_compute_algo, reduce_amax, use_emha, ub_tp_comm_overlap, use_flash_attention, seq_len_interpolation_factor, rotary_base)