class_docstr: Modified pretraining for depth-recurrent models. Only works with models that expose the ScriptableRecurrentLM interface.
class_docstr_tok: ["Modified","pretraining","for","depth-recurrent","models",".","Only","works","with","models","that","expose","the","ScriptableRecurrentLM","interface","."]
code: def __init__(self, encoder, cfg_arch):
      super().__init__()
      self.cfg = cfg_arch
  
      self.encoder = encoder
      if not cfg_arch.skip_head_transform:
          self.prediction_head = PredictionHeadComponent(cfg_arch)
      else:
          self.prediction_head = torch.nn.Linear(
              cfg_arch.hidden_size,
              cfg_arch.embedding.embedding_dim,
              bias=cfg_arch.use_bias,
          )
  
      if self.cfg.tie_weights:
          self.decoder = torch.nn.Linear(
              cfg_arch.embedding.embedding_dim,
              cfg_arch.embedding.vocab_size,
              bias=cfg_arch.decoder_bias,
          )
          self.decoder.weight = (
              self.encoder.embedding.word_embedding.weight
          )
      else:
          self.decoder = torch.nn.Linear(
              cfg_arch.hidden_size,
              cfg_arch.embedding.vocab_size,
              bias=cfg_arch.decoder_bias,
          )
  
      self.loss_fn = torch.nn.CrossEntropyLoss(reduction="mean")
      self.sparse_prediction = self.cfg.sparse_prediction
      self.vocab_size = cfg_arch.embedding.vocab_size
  
      for name, module in self.named_modules():
          _init_module(
              module,
              self.cfg.init.type,
              self.cfg.init.std,
              self.cfg.hidden_size,
              self.cfg.recurrent_layers,
          )
  
      if cfg_arch.training_scheme == "bptt-deepthinking":
          self._forward_method = self._forward_deepthinking
      elif cfg_arch.training_scheme == "fixed-recurrence":
          self._forward_method = self._forward_fixed
      else:
          raise ValueError(
              f"Invalid training scheme {cfg_arch.training_scheme} given."
          )
  
imports: import torch
  from typing import Optional
  from random import randrange
  from transformers import PretrainedConfig
  from .components import EmbeddingComponent, AttentionComponent, FFNComponent, PredictionHeadComponent, get_extended_attention_mask, _get_norm_fn, _get_nonlin_fn, _init_module
label_desc: Use image node_id 1 to create a new BPTTforPreTraining object from inherited base classes: torch.nn with example: obj = BPTTforPreTraining(encoder, cfg_arch)