              self._run_training_cycle(model, trainer)
          except KeyboardInterrupt:
              try:
                  logger.debug("Keyboard Interrupt Caught. Saving Weights and exiting")
                  model.save(on_exit=True)
                  trainer.clear_tensorboard()
              except KeyboardInterrupt:
                  logger.info("Saving model weights has been cancelled!")
              sys.exit(0)
          except Exception as err:
 
</s> remove         self.add_item(
            section=section,
            title="save_optimizer",
            datatype=str,
            group="optimizer",
            default="exit",
            fixed=False,
            gui_radio=True,
            choices=["never", "always", "exit"],
            info="When to save the Optimizer Weights. Saving the optimizer weights is not "
                 "necessary and will increase the model file size 3x (and by extension the amount "
                 "of time it takes to save the model). However, it can be useful to save these "
                 "weights if you want to guarantee that a resumed model carries off exactly from "
                 "where it left off, rather than spending a few hundred iterations catching up."
                 "\n\t never - Don't save optimizer weights."
                 "\n\t always - Save the optimizer weights at every save iteration. Model saving "
                 "will take longer, due to the increased file size, but you will always have the "
                 "last saved optimizer state in your model file."
                 "\n\t exit - Only save the optimizer weights when explicitly terminating a "
                 "model. This can be when the model is actively stopped or when the target "
                 "iterations are met. Note: If the training session ends because of another "
                 "reason (e.g. power outage, Out of Memory Error, NaN detected) then the "
                 "optimizer weights will NOT be saved.")
 </s> add  </s> remove         self.save(force_save_optimizer=True)
 </s> add         # self.save(force_save_optimizer=True)
        # TODO Re-enable saving optimizer state when h5 bug fixed:
        # File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
        # File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
        # File "h5py/h5d.pyx", line 87, in h5py.h5d.create
        # ValueError: Unable to create dataset (name already exists)
 </s> remove         self._io = IO(self, model_dir, self._is_predict, self.config["save_optimizer"])
 </s> add         # self._io = IO(self, model_dir, self._is_predict, self.config["save_optimizer"])
        # TODO - Re-enable saving of optimizer once this bug is fixed:
        # File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
        # File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
        # File "h5py/h5d.pyx", line 87, in h5py.h5d.create
        # ValueError: Unable to create dataset (name already exists)

        self._io = IO(self, model_dir, self._is_predict, "never")
                model.save(is_exit=True)
              self._run_training_cycle(model, trainer)
          except KeyboardInterrupt:
              try:
                  logger.debug("Keyboard Interrupt Caught. Saving Weights and exiting")
                model.save(is_exit=True)
                  trainer.clear_tensorboard()
              except KeyboardInterrupt:
                  logger.info("Saving model weights has been cancelled!")
              sys.exit(0)
          except Exception as err:
keep keep keep keep replace keep keep keep keep keep
 Disable saving optimizer state option 
scripts/train.py