class_docstr: Left-to-right Transformer language model.
code: def __init__(self, cfg: DictConfig, trainer: Trainer = None):
      # Get global rank and total number of GPU workers for IterableDataset partitioning, if applicable
      self.world_size = 1
      if trainer is not None:
          self.world_size = trainer.num_nodes * trainer.num_devices
  
      cfg = model_utils.convert_model_config_to_dict_config(cfg)
      cfg = model_utils.maybe_update_config_version(cfg)
  
      # Instantiates tokenizer and register to be saved with NeMo Model archive
      # After this call, ther will be self.tokenizer which can convert between tokens and token_ids.
      self.setup_tokenizer(
          tokenizer_name=cfg.tokenizer.get("tokenizer_name", "yttm"),
          tokenizer_model=cfg.tokenizer.get("tokenizer_model", None),
          vocab_file=cfg.tokenizer.get("vocab_file", None),
          bpe_dropout=cfg.tokenizer.get("bpe_dropout", 0.0),
          special_tokens=OmegaConf.to_container(
              cfg.tokenizer.special_tokens
          )
          if cfg.tokenizer.get("special_tokens", None)
          else None,
      )
  
      # init superclass
      super().__init__(cfg=cfg, trainer=trainer)
  
      # make vocabulary size divisible by 8 for fast fp16 training
      vocab_size = 8 * math.ceil(self.tokenizer.vocab_size / 8)
  
      # encoder from NeMo, Megatron-LM, or HuggingFace
      encoder_cfg_dict = OmegaConf.to_container(cfg.get("encoder"))
      encoder_cfg_dict["vocab_size"] = vocab_size
      library = encoder_cfg_dict.pop("library", "nemo")
      model_name = encoder_cfg_dict.pop("model_name", None)
      pretrained = encoder_cfg_dict.pop("pretrained", False)
      self.encoder = get_transformer(
          library=library,
          model_name=model_name,
          pretrained=pretrained,
          config_dict=encoder_cfg_dict,
          encoder=True,
          pre_ln_final_layer_norm=encoder_cfg_dict.get(
              "pre_ln_final_layer_norm",
              encoder_cfg_dict.get("pre_ln", True),
          ),
      )
  
      self.log_softmax = TokenClassifier(
          hidden_size=self.encoder.hidden_size,
          num_classes=vocab_size,
          activation=cfg.head.activation,
          log_softmax=cfg.head.log_softmax,
          dropout=cfg.head.dropout,
          use_transformer_init=cfg.head.use_transformer_init,
      )
  
      # tie weights of embedding and softmax matrices
      self.log_softmax.mlp.layer0.weight = (
          self.encoder.embedding.token_embedding.weight
      )
  
      std_init_range = 1 / self.encoder.hidden_size**0.5
  
      # initialize weights if not using pretrained encoder
      if not self._cfg.encoder.get("pretrained", False):
          self.encoder.apply(
              lambda module: transformer_weights_init(
                  module, std_init_range
              )
          )
  
      self.log_softmax.apply(
          lambda module: transformer_weights_init(
              module, std_init_range
          )
      )
  
      self.loss_fn = SmoothedCrossEntropyLoss(
          pad_id=self.tokenizer.pad_id,
          label_smoothing=cfg.label_smoothing,
      )
      self.eval_loss_fn = SmoothedCrossEntropyLoss(
          pad_id=self.tokenizer.pad_id
      )
      self.eval_loss = GlobalAverageLossMetric(
          dist_sync_on_step=False, take_avg_loss=True
      )
      self.eval_ppl = SequencePerplexity()
  
imports: import json
  import math
  from typing import Dict, Optional
  import numpy
  import torch
  import torch.utils.data
  from omegaconf import DictConfig, OmegaConf
  from pytorch_lightning import Trainer
  from nemo.collections.common.losses import SmoothedCrossEntropyLoss
  from nemo.collections.common.metrics import GlobalAverageLossMetric
  from nemo.collections.common.parts import transformer_weights_init
  from nemo.collections.nlp.data import SentenceDataset, TarredSentenceDataset
  from nemo.collections.nlp.metrics import SequencePerplexity
  from nemo.collections.nlp.modules.common import TokenClassifier
  from nemo.collections.nlp.modules.common.lm_utils import get_transformer
  from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer
  from nemo.core.classes.common import typecheck
  from nemo.core.classes.modelPT import ModelPT
  from nemo.utils import logging, model_utils
label_desc: Use image node_id 1 to create a new TransformerLMModel object from inherited base classes: ModelPT with example: obj = TransformerLMModel(cfg, trainer)