              for side, side_loss in loss.items():
                  self.store_history(side, side_loss)
                  self.log_tensorboard(side, side_loss)
  
              if not self.pingpong.active:
                  self.print_loss(loss)
              else:
                  for key, val in loss.items():
                      self.pingpong.loss[key] = val
                  self.print_loss(self.pingpong.loss)
  
 
</s> remove                 self.store_history(side, side_loss)
                self.log_tensorboard(side, side_loss)
 </s> add                 self._store_history(side, side_loss)
                self._log_tensorboard(side, side_loss) </s> remove                     self.pingpong.loss[key] = val
                self.print_loss(self.pingpong.loss)
 </s> add                     self._pingpong.loss[key] = val
                self.__print_loss(self._pingpong.loss) </s> remove             self.model.state.increment_iterations()
 </s> add             self._model.state.increment_iterations() </s> remove                     self.timelapse.get_sample(side, timelapse_kwargs)
 </s> add                     self._timelapse.get_sample(side, timelapse_kwargs) </s> remove                      {key: val for key, val in training_opts.items() if key != "landmarks"},
                     bool(training_opts.get("landmarks", None)), config)
 </s> add                      {key: val
                      for key, val in training_opts.items() if key not in ("landmarks", "masks")},
                     {key: len(val)
                      for key, val in training_opts.get("landmarks", dict()).items()},
                     {key: len(val) for key, val in training_opts.get("masks", dict()).items()},
                     config)
            if not self._pingpong.active:
                self.__print_loss(loss)
              for side, side_loss in loss.items():
                  self.store_history(side, side_loss)
                  self.log_tensorboard(side, side_loss)
  
            if not self._pingpong.active:
                self.__print_loss(loss)
            if not self._pingpong.active:
                self.__print_loss(loss)
              else:
                  for key, val in loss.items():
                      self.pingpong.loss[key] = val
                  self.print_loss(self.pingpong.loss)
  
keep keep keep keep replace replace keep keep keep keep keep
 Smart Masks - Training Implementation (#914)

* Smart Masks - Training

- Reinstate smart mask training code
- Reinstate mask_type back to model.config
- change 'replicate_input_mask to 'learn_mask'
- Add learn mask option
- Add mask loading from alignments to plugins.train.trainer
- Add mask_blur and mask threshold options
- _base.py - Pass mask options through training_opts dict
- plugins.train.model - check for mask_type not None for learn_mask and penalized_mask_loss
- Limit alignments loading to just those faces that appear in the training folder
- Raise error if not all training images have an alignment, and alignment file is required
- lib.training_data - Mask generation code
- lib.faces_detect - cv2 dimension stripping bugfix
- Remove cv2 linting code

* Update mask helptext in cli.py

* Fix Warp to Landmarks
Remove SHA1 hashing from training data

* Update mask training config

* Capture missing masks at training init

* lib.image.read_image_batch - Return filenames with batch for ordering

* scripts.train - Documentation

* plugins.train.trainer - documentation

* Ensure backward compatibility.
Fix convert for new predicted masks

* Update removed masks to components for legacy models. 
plugins/train/trainer/_base.py