class_docstr: Pretraining version with optional prediction head and variant for sparse prediction.
class_docstr_tok: ["Pretraining","version","with","optional","prediction","head","and","variant","for","sparse","prediction","."]
code: def __init__(self, config):
      super().__init__(config)
      self.cfg = OmegaConf.create(config.arch)
  
      self.encoder = ScriptableLM(config)
  
      if not self.cfg.skip_head_transform:
          self.prediction_head = PredictionHeadComponent(self.cfg)
      else:
          self.prediction_head = (
              torch.nn.Identity()
          )  # from linear in old version
  
      self.decoder = torch.nn.Linear(
          self.cfg.embedding.embedding_dim,
          self.cfg.embedding.vocab_size,
          bias=self.cfg.decoder_bias,
      )
      self.decoder.weight = self.encoder.embedding.word_embedding.weight
  
      self.loss_fn = torch.nn.CrossEntropyLoss()
      self.sparse_prediction = self.cfg.sparse_prediction
  
      self._init_weights()
  
imports: import torch
  from transformers import PretrainedConfig, PreTrainedModel
  from transformers import AutoConfig, AutoModel, AutoModelForMaskedLM, AutoModelForSequenceClassification, AutoModelForTokenClassification
  from typing import Optional
  from omegaconf import OmegaConf
  from .fff import FFF
  from .components import _get_norm_fn, _get_nonlin_fn, EmbeddingComponent, PoolingComponent, PredictionHeadComponent, GLU, get_extended_attention_mask, _init_module
  from .attention import get_attention_mechanism
label_desc: Use image node_id 1 to create a new ScriptableLMForPreTraining object from inherited base classes: PreTrainedModel with example: obj = ScriptableLMForPreTraining(config)