              sleep(1)  # Let preview instructions flush out to logger
              logger.debug("Commencing Training")
              logger.info("Loading data, this may take a while...")
  
              if self.args.allow_growth:
                  self.set_tf_allow_growth()
              model = self.load_model()
              trainer = self.load_trainer(model)
              self.run_training_cycle(model, trainer)
          except KeyboardInterrupt:
              try:
                  logger.debug("Keyboard Interrupt Caught. Saving Weights and exiting")
                  model.save_models()
                  trainer.clear_tensorboard()
 
</s> remove     def training(self):
        """ The training process to be run inside a thread """
 </s> add     def _training(self):
        """ The training process to be run inside a thread. """ </s> remove         logger.debug("Ended Training thread")
 </s> add         logger.debug("Ended training thread") </s> remove                         self.save_now = True
 </s> add                         self._save_now = True </s> remove         self.stop = True
 </s> add         self._stop = True </s> remove                 self.save_now = False
 </s> add                 self._save_now = False
            if self._args.allow_growth:
                self._set_tf_allow_growth()
            model = self._load_model()
            trainer = self._load_trainer(model)
            self._run_training_cycle(model, trainer)
              sleep(1)  # Let preview instructions flush out to logger
              logger.debug("Commencing Training")
              logger.info("Loading data, this may take a while...")
  
            if self._args.allow_growth:
                self._set_tf_allow_growth()
            model = self._load_model()
            trainer = self._load_trainer(model)
            self._run_training_cycle(model, trainer)
            if self._args.allow_growth:
                self._set_tf_allow_growth()
            model = self._load_model()
            trainer = self._load_trainer(model)
            self._run_training_cycle(model, trainer)
            if self._args.allow_growth:
                self._set_tf_allow_growth()
            model = self._load_model()
            trainer = self._load_trainer(model)
            self._run_training_cycle(model, trainer)
            if self._args.allow_growth:
                self._set_tf_allow_growth()
            model = self._load_model()
            trainer = self._load_trainer(model)
            self._run_training_cycle(model, trainer)
            if self._args.allow_growth:
                self._set_tf_allow_growth()
            model = self._load_model()
            trainer = self._load_trainer(model)
            self._run_training_cycle(model, trainer)
          except KeyboardInterrupt:
              try:
                  logger.debug("Keyboard Interrupt Caught. Saving Weights and exiting")
                  model.save_models()
                  trainer.clear_tensorboard()
keep keep keep keep replace replace replace replace replace keep keep keep keep keep
 Smart Masks - Training Implementation (#914)

* Smart Masks - Training

- Reinstate smart mask training code
- Reinstate mask_type back to model.config
- change 'replicate_input_mask to 'learn_mask'
- Add learn mask option
- Add mask loading from alignments to plugins.train.trainer
- Add mask_blur and mask threshold options
- _base.py - Pass mask options through training_opts dict
- plugins.train.model - check for mask_type not None for learn_mask and penalized_mask_loss
- Limit alignments loading to just those faces that appear in the training folder
- Raise error if not all training images have an alignment, and alignment file is required
- lib.training_data - Mask generation code
- lib.faces_detect - cv2 dimension stripping bugfix
- Remove cv2 linting code

* Update mask helptext in cli.py

* Fix Warp to Landmarks
Remove SHA1 hashing from training data

* Update mask training config

* Capture missing masks at training init

* lib.image.read_image_batch - Return filenames with batch for ordering

* scripts.train - Documentation

* plugins.train.trainer - documentation

* Ensure backward compatibility.
Fix convert for new predicted masks

* Update removed masks to components for legacy models. 
scripts/train.py