class_docstr: Bert-based module for Inverse Cloze task.
class_docstr_tok: ["Bert-based","module","for","Inverse","Cloze","task","."]
docstr: Initialize the state from a pretrained BERT model on iteration zero of ICT pretraining
docstr_tok: ["Initialize","the","state","from","a","pretrained","BERT","model","on","iteration","zero","of","ICT","pretraining"]
code: def init_state_dict_from_bert(self):
      """Initialize the state from a pretrained BERT model on iteration zero of ICT pretraining"""
      args = get_args()
      tracker_filename = get_checkpoint_tracker_filename(args.bert_load)
      if not os.path.isfile(tracker_filename):
          raise FileNotFoundError("Could not find BERT load for ICT")
      with open(tracker_filename, "r") as f:
          iteration = int(f.read().strip())
          assert iteration > 0
  
      checkpoint_name = get_checkpoint_name(
          args.bert_load, iteration, False
      )
      if mpu.get_data_parallel_rank() == 0:
          print(
              "global rank {} is loading checkpoint {}".format(
                  torch.distributed.get_rank(), checkpoint_name
              )
          )
  
      try:
          state_dict = torch.load(checkpoint_name, map_location="cpu")
      except BaseException:
          raise ValueError("Could not load checkpoint")
  
      # load the LM state dict into each model
      model_dict = state_dict["model"]["language_model"]
      self.query_model.language_model.load_state_dict(model_dict)
      self.block_model.language_model.load_state_dict(model_dict)
  
      # give each model the same ict_head to begin with as well
      query_ict_head_state_dict = self.state_dict_for_save_checkpoint()[
          self._query_key
      ]["ict_head"]
      self.block_model.ict_head.load_state_dict(
          query_ict_head_state_dict
      )
  
imports: import os
  import torch
  from megatron import get_args, print_rank_0
  from megatron.checkpointing import get_checkpoint_tracker_filename, get_checkpoint_name
  from megatron.model import BertModel
  from .module import MegatronModule
  from megatron.core import mpu
  from megatron.model.enums import AttnMaskType
  from megatron.model.utils import get_linear_layer
  from megatron.model.utils import init_method_normal
  from megatron.model.language_model import get_language_model
  from megatron.model.utils import scaled_init_method_normal
  from megatron.model.bert_model import bert_extended_attention_mask, bert_position_ids
label_desc: Use image node_id 7 for calling the ICTBertModel obj's underlying member method code with example usage: obj.init_state_dict_from_bert() without return types