class_docstr: saves a checkpoint after each epoch as 'last_state' (can be loaded to continue interrupted training).
  saves the top-k (k=cf.save_n_models) ranked epochs. In inference, predictions of multiple epochs can be ensembled to improve performance.
class_docstr_tok: ["saves","a","checkpoint","after","each","epoch","as","'last_state","'","(","can","be","loaded","to","continue","interrupted","training",")",".","saves","the","top-k","(","k=cf.save_n_models",")","ranked","epochs",".","In","inference",",","predictions","of","multiple","epochs","can","be","ensembled","to","improve","performance","."]
code: def run_model_selection(self, net, optimizer, monitor_metrics, epoch):
      # take the mean over all selection criteria in each epoch
      non_nan_scores = np.mean(
          np.array(
              [
                  [
                      0 if ii is None else ii
                      for ii in monitor_metrics["val"][sc]
                  ]
                  for sc in self.cf.model_selection_criteria
              ]
          ),
          0,
      )
      epochs_scores = [ii for ii in non_nan_scores[1:]]
      # ranking of epochs according to model_selection_criterion
      epoch_ranking = (
          np.argsort(epochs_scores)[::-1] + 1
      )  # epochs start at 1
      # if set in configs, epochs < min_save_thresh are discarded from saving process.
      epoch_ranking = epoch_ranking[
          epoch_ranking >= self.cf.min_save_thresh
      ]
  
      # check if current epoch is among the top-k epchs.
      if epoch in epoch_ranking[: self.cf.save_n_models]:
          save_dir = os.path.join(
              self.cf.fold_dir, "{}_best_checkpoint".format(epoch)
          )
          if not os.path.exists(save_dir):
              os.mkdir(save_dir)
  
          torch.save(
              net.state_dict(), os.path.join(save_dir, "params.pth")
          )
          with open(
              os.path.join(save_dir, "monitor_metrics.pickle"), "wb"
          ) as handle:
              pickle.dump(monitor_metrics, handle)
          # save epoch_ranking to keep info for inference.
          np.save(
              os.path.join(self.cf.fold_dir, "epoch_ranking"),
              epoch_ranking[: self.cf.save_n_models],
          )
          np.save(
              os.path.join(save_dir, "epoch_ranking"),
              epoch_ranking[: self.cf.save_n_models],
          )
  
          self.logger.info(
              "saving current epoch {} at rank {}".format(
                  epoch, np.argwhere(epoch_ranking == epoch)
              )
          )
          # delete params of the epoch that just fell out of the top-k epochs.
          for se in [
              int(ii.split("_")[0])
              for ii in os.listdir(self.cf.fold_dir)
              if "best_checkpoint" in ii
          ]:
              if se in epoch_ranking[self.cf.save_n_models :]:
                  subprocess.call(
                      "rm -rf {}".format(
                          os.path.join(
                              self.cf.fold_dir,
                              "{}_best_checkpoint".format(se),
                          )
                      ),
                      shell=True,
                  )
                  self.logger.info(
                      "deleting epoch {} at rank {}".format(
                          se, np.argwhere(epoch_ranking == se)
                      )
                  )
  
      state = {
          "epoch": epoch,
          "state_dict": net.state_dict(),
          "optimizer": optimizer.state_dict(),
      }
  
      # save checkpoint of current epoch.
      save_dir = os.path.join(
          self.cf.fold_dir, "last_checkpoint".format(epoch)
      )
      if not os.path.exists(save_dir):
          os.mkdir(save_dir)
      torch.save(state, os.path.join(save_dir, "params.pth"))
      np.save(
          os.path.join(save_dir, "epoch_ranking"),
          epoch_ranking[: self.cf.save_n_models],
      )
      with open(
          os.path.join(save_dir, "monitor_metrics.pickle"), "wb"
      ) as handle:
          pickle.dump(monitor_metrics, handle)
  
imports: import numpy
  import logging
  import subprocess
  import os
  import torch
  from collections import OrderedDict
  import plotting
  import sys
  import importlib.util
  import pandas
  import pickle
label_desc: Use image node_id 2 for calling the ModelSelector obj's underlying member method code with example usage: obj.run_model_selection(net, optimizer, monitor_metrics, epoch) without return types