class_docstr: the full GPT language model, with a context size of block_size
class_docstr_tok: ["the","full","GPT","language","model",",","with","a","context","size","of","block_size"]
code: def __init__(
      self,
      vocab_size,
      weight_decay=0.1,
      betas=(0.9, 0.95),
      learning_rate=6e-4,
      n_embd=512,
      block_size=128,
      n_layer=8,
      n_head=8,
      resid_pdrop=0.1,
      attn_pdrop=0.1,
      mlp_pdrop=0.1,
      attention="scaled_dot_product",
      hidden_layer_multiplier=4,
      warmup_tokens=20,
      final_tokens=1000,
  ):
      super().__init__()
  
      # auto creates self.hparams from the method signature
      self.save_hyperparameters()
  
      # A list of the encoder or decoder blocks which constitute the Transformer.
      xformer_config = [
          {
              "reversible": False,  # Turn on to test the effect of using reversible layers
              "block_type": "encoder",
              "num_layers": self.hparams.n_layer,
              "dim_model": self.hparams.n_embd,
              "residual_norm_style": "post",
              "position_encoding_config": {
                  "name": "vocab",
                  "seq_len": self.hparams.block_size,
                  "vocab_size": self.hparams.vocab_size,
              },
              "multi_head_config": {
                  "num_heads": self.hparams.n_head,
                  "residual_dropout": self.hparams.resid_pdrop,
                  "use_rotary_embeddings": True,
                  "attention": {
                      "name": self.hparams.attention,
                      "dropout": self.hparams.attn_pdrop,
                      "causal": True,
                      "seq_len": self.hparams.block_size,
                      "num_rules": self.hparams.n_head,
                  },
              },
              "feedforward_config": {
                  "name": "FusedMLP",  # Use MLP if Triton is not available
                  "dropout": self.hparams.mlp_pdrop,
                  "activation": "gelu",
                  "hidden_layer_multiplier": self.hparams.hidden_layer_multiplier,
              },
          }
      ]
  
      config = xFormerConfig(xformer_config)
      config.weight_init = "small"
      self.model = xFormer.from_config(config)
  
      # decoder head
      self.ln_f = nn.LayerNorm(self.hparams.n_embd)
      self.head = nn.Linear(
          self.hparams.n_embd, self.hparams.vocab_size, bias=False
      )
  
      self.block_size = self.hparams.block_size
      self.apply(self._init_weights)
  
      self._tokens_seen = 0
  
imports: import math
  import os
  import pytorch_lightning
  import torch
  import torch.nn
  from pytorch_lightning import Trainer, seed_everything
  from pytorch_lightning.utilities import rank_zero_info
  from torch.nn import functional
  from torch.utils.data import DataLoader, Dataset, RandomSampler
  from xformers.factory.model_factory import xFormer, xFormerConfig
label_desc: Use image node_id 1 to create a new GPT object from inherited base classes: pl with example: obj = GPT(vocab_size, weight_decay, betas, learning_rate, n_embd, block_size, n_layer, n_head, resid_pdrop, attn_pdrop, mlp_pdrop, attention, hidden_layer_multiplier, warmup_tokens, final_tokens)