jiant/trainer.py
def _save_checkpoint(self, training_state, phase="pretrain", new_best=False, tasks=None):
        """
        Parameters
        ----------
        training_state: An object containing trainer state (step number, etc.), to be saved.
        phase: Usually 'pretrain' or 'target_train'.
        new_best: If true, the saved checkpoint will be marked with .best_macro, and
            potentially used later when switching from pretraining to target task training.
        """
        if not self._serialization_dir:
            raise ConfigurationError(
                "serialization_dir not specified - cannot "
                "restore a model without a directory path."
            )

        val_pass = training_state["validation_pass"]
        if new_best:
            best_str = ".best"
        else:
            best_str = ""

        task_dir_name = ""

        if phase == "target_train":
            # We only pass in one task at a time during target train phase.
            assert len(tasks) == 1
            task_dir_name = tasks[0].name

        model_path = os.path.join(
            self._serialization_dir,
            task_dir_name,
            "model_state_{}_val_{}{}.th".format(phase, val_pass, best_str),
        )

        if self._records_pickle_path:
            records_dict = get_records_dict(self._records_pickle_path)
            records_dict['last_checkpoint'] = model_path
            write_records_dict(records_dict, self._records_pickle_path)


        model_state = self._model.state_dict()

        # Skip non-trainable params, like the main ELMo params.
        for name, param in self._model.named_parameters():
            if not param.requires_grad:
                del model_state[name]

        task_states = {}
        for task_name, task_info in self._task_infos.items():
            task_states[task_name] = {}
            task_states[task_name]["total_batches_trained"] = task_info["total_batches_trained"]
            task_states[task_name]["stopped"] = task_info["stopped"]
        task_states["global"] = {}
        task_states["global"]["optimizer"] = self._optimizer.state_dict()
        # NOTE(Alex): AllenNLP wrapper doesn't expose scheduler state dict methods
        task_states["global"]["scheduler"] = self._scheduler.lr_scheduler.state_dict()

        metric_states = {}
        for metric_name, metric_info in self._metric_infos.items():
            metric_states[metric_name] = {}
            metric_states[metric_name]["hist"] = metric_info["hist"]
            metric_states[metric_name]["stopped"] = metric_info["stopped"]
            metric_states[metric_name]["best"] = metric_info["best"]

        torch.save(
            task_states,
            os.path.join(
                self._serialization_dir,
                task_dir_name,
                "task_state_{}_val_{}{}.th".format(phase, val_pass, best_str),
            ),
        )
        torch.save(
            metric_states,
            os.path.join(
                self._serialization_dir,
                task_dir_name,
                "metric_state_{}_val_{}{}.th".format(phase, val_pass, best_str),
            ),
        )
        torch.save(model_state, model_path)
        torch.save(
            training_state,
            os.path.join(
                self._serialization_dir,
                task_dir_name,
                "training_state_{}_val_{}{}.th".format(phase, val_pass, best_str),
            ),
        )
        if new_best:
            self._unmark_previous_best(phase, val_pass, task_dir_name)

        if not self._keep_all_checkpoints:
            self._delete_old_checkpoints(phase, val_pass, task_dir_name)

        log.info("Saved checkpoints to %s", self._serialization_dir)

        Parameters
        ----------
        training_state: An object containing trainer state (step number, etc.), to be saved.
        phase: Usually 'pretrain' or 'target_train'.
        new_best: If true, the saved checkpoint will be marked with .best_macro, and
            potentially used later when switching from pretraining to target task training.
        
Parameters
training_state: An object containing trainer state (step number, etc.), to be saved.
phase: Usually 'pretrain' or 'target_train'.
new_best: If true, the saved checkpoint will be marked with .best_macro, and
potentially used later when switching from pretraining to target task training.
Parameters training_state : An object containing trainer state ( step number etc . ) to be saved . phase : Usually ' pretrain ' or ' target_train ' . new_best : If true the saved checkpoint will be marked with . best_macro and potentially used later when switching from pretraining to target task training .
def _save_checkpoint(self, training_state, phase="pretrain", new_best=False, tasks=None):
        if not self._serialization_dir:
            raise ConfigurationError(
                "serialization_dir not specified - cannot "
                "restore a model without a directory path."
            )
        val_pass = training_state["validation_pass"]
        if new_best:
            best_str = ".best"
        else:
            best_str = ""
        task_dir_name = ""
        if phase == "target_train":
            assert len(tasks) == 1
            task_dir_name = tasks[0].name
        model_path = os.path.join(
            self._serialization_dir,
            task_dir_name,
            "model_state_{}_val_{}{}.th".format(phase, val_pass, best_str),
        )
        if self._records_pickle_path:
            records_dict = get_records_dict(self._records_pickle_path)
            records_dict['last_checkpoint'] = model_path
            write_records_dict(records_dict, self._records_pickle_path)
        model_state = self._model.state_dict()
        for name, param in self._model.named_parameters():
            if not param.requires_grad:
                del model_state[name]
        task_states = {}
        for task_name, task_info in self._task_infos.items():
            task_states[task_name] = {}
            task_states[task_name]["total_batches_trained"] = task_info["total_batches_trained"]
            task_states[task_name]["stopped"] = task_info["stopped"]
        task_states["global"] = {}
        task_states["global"]["optimizer"] = self._optimizer.state_dict()
        task_states["global"]["scheduler"] = self._scheduler.lr_scheduler.state_dict()
        metric_states = {}
        for metric_name, metric_info in self._metric_infos.items():
            metric_states[metric_name] = {}
            metric_states[metric_name]["hist"] = metric_info["hist"]
            metric_states[metric_name]["stopped"] = metric_info["stopped"]
            metric_states[metric_name]["best"] = metric_info["best"]
        torch.save(
            task_states,
            os.path.join(
                self._serialization_dir,
                task_dir_name,
                "task_state_{}_val_{}{}.th".format(phase, val_pass, best_str),
            ),
        )
        torch.save(
            metric_states,
            os.path.join(
                self._serialization_dir,
                task_dir_name,
                "metric_state_{}_val_{}{}.th".format(phase, val_pass, best_str),
            ),
        )
        torch.save(model_state, model_path)
        torch.save(
            training_state,
            os.path.join(
                self._serialization_dir,
                task_dir_name,
                "training_state_{}_val_{}{}.th".format(phase, val_pass, best_str),
            ),
        )
        if new_best:
            self._unmark_previous_best(phase, val_pass, task_dir_name)
        if not self._keep_all_checkpoints:
            self._delete_old_checkpoints(phase, val_pass, task_dir_name)
        log.info("Saved checkpoints to %s", self._serialization_dir)
def _save_checkpoint ( self , training_state , phase = "pretrain" , new_best = False , tasks = None ) : if not self . _serialization_dir : raise ConfigurationError ( "serialization_dir not specified - cannot " "restore a model without a directory path." ) val_pass = training_state [ "validation_pass" ] if new_best : best_str = ".best" else : best_str = "" task_dir_name = "" if phase == "target_train" : assert len ( tasks ) == 1 task_dir_name = tasks [ 0 ] . name model_path = os . path . join ( self . _serialization_dir , task_dir_name , "model_state_{}_val_{}{}.th" . format ( phase , val_pass , best_str ) , ) if self . _records_pickle_path : records_dict = get_records_dict ( self . _records_pickle_path ) records_dict [ 'last_checkpoint' ] = model_path write_records_dict ( records_dict , self . _records_pickle_path ) model_state = self . _model . state_dict ( ) for name , param in self . _model . named_parameters ( ) : if not param . requires_grad : del model_state [ name ] task_states = { } for task_name , task_info in self . _task_infos . items ( ) : task_states [ task_name ] = { } task_states [ task_name ] [ "total_batches_trained" ] = task_info [ "total_batches_trained" ] task_states [ task_name ] [ "stopped" ] = task_info [ "stopped" ] task_states [ "global" ] = { } task_states [ "global" ] [ "optimizer" ] = self . _optimizer . state_dict ( ) task_states [ "global" ] [ "scheduler" ] = self . _scheduler . lr_scheduler . state_dict ( ) metric_states = { } for metric_name , metric_info in self . _metric_infos . items ( ) : metric_states [ metric_name ] = { } metric_states [ metric_name ] [ "hist" ] = metric_info [ "hist" ] metric_states [ metric_name ] [ "stopped" ] = metric_info [ "stopped" ] metric_states [ metric_name ] [ "best" ] = metric_info [ "best" ] torch . save ( task_states , os . path . join ( self . _serialization_dir , task_dir_name , "task_state_{}_val_{}{}.th" . format ( phase , val_pass , best_str ) , ) , ) torch . save ( metric_states , os . path . join ( self . _serialization_dir , task_dir_name , "metric_state_{}_val_{}{}.th" . format ( phase , val_pass , best_str ) , ) , ) torch . save ( model_state , model_path ) torch . save ( training_state , os . path . join ( self . _serialization_dir , task_dir_name , "training_state_{}_val_{}{}.th" . format ( phase , val_pass , best_str ) , ) , ) if new_best : self . _unmark_previous_best ( phase , val_pass , task_dir_name ) if not self . _keep_all_checkpoints : self . _delete_old_checkpoints ( phase , val_pass , task_dir_name ) log . info ( "Saved checkpoints to %s" , self . _serialization_dir )
Parameters
training_state: An object containing trainer state (step number, etc.
Parameters training_state : An object containing trainer state ( step number etc .
"""
        Parameters
        ----------
        training_state: An object containing trainer state (step number, etc.), to be saved.
        phase: Usually 'pretrain' or 'target_train'.
        new_best: If true, the saved checkpoint will be marked with .best_macro, and
            potentially used later when switching from pretraining to target task training.
        """ # We only pass in one task at a time during target train phase. # Skip non-trainable params, like the main ELMo params. # NOTE(Alex): AllenNLP wrapper doesn't expose scheduler state dict methods
self
training_state
phase
new_best
tasks