          logger.trace("Updating loss history: '%s'", side)
          self.model.history[side].append(loss[0])  # Either only loss or total loss
          logger.trace("Updated loss history: '%s'", side)
  
      def log_tensorboard(self, side, loss):
          """ Log loss to TensorBoard log """
          if not self.tensorboard:
              return
          logger.trace("Updating TensorBoard log: '%s'", side)
          logs = {log[0]: log[1]
                  for log in zip(self.model.state.loss_names[side], loss)}
          self.tensorboard[side].on_batch_end(self.model.state.iterations, logs)
 
</s> remove         self.model.history[side].append(loss[0])  # Either only loss or total loss
 </s> add         self._model.history[side].append(loss[0])  # Either only loss or total loss </s> remove                 for log in zip(self.model.state.loss_names[side], loss)}
        self.tensorboard[side].on_batch_end(self.model.state.iterations, logs)
 </s> add                 for log in zip(self._model.state.loss_names[side], loss)}
        self._tensorboard[side].on_batch_end(self._model.state.iterations, logs) </s> remove     def store_history(self, side, loss):
        """ Store the history of this step """
 </s> add     def _store_history(self, side, loss):
        """ Store the loss for this step into :attr:`model.history`.

        Parameters
        ----------
        side: {"a", "b"}
            The side to store the loss for
        loss: list
            The list of loss ``floats`` for this side
        """ </s> remove         """ Indicate training end to Tensorboard """
        if not self.tensorboard:
 </s> add         """ Stop Tensorboard logging.

        Tensorboard logging needs to be explicitly shutdown on training termination. Called from
        :class:`scripts.train.Train` when training is stopped.
         """
        if not self._tensorboard: </s> remove     def print_loss(self, loss):
        """ Override for specific model loss formatting """
 </s> add     def __print_loss(self, loss):
        """ Outputs the loss for the current iteration to the console.

        Parameters
        ----------
        loss: dict
            The loss for each side. The dictionary should contain 2 keys ("a" and "b") with the
            values being a list of loss values for the current iteration corresponding to
            each side.
         """
    def _log_tensorboard(self, side, loss):
        """ Log current loss to Tensorboard log files

        Parameters
        ----------
        side: {"a", "b"}
            The side to store the loss for
        loss: list
            The list of loss ``floats`` for this side
        """
        if not self._tensorboard:
          logger.trace("Updating loss history: '%s'", side)
          self.model.history[side].append(loss[0])  # Either only loss or total loss
          logger.trace("Updated loss history: '%s'", side)
  
    def _log_tensorboard(self, side, loss):
        """ Log current loss to Tensorboard log files

        Parameters
        ----------
        side: {"a", "b"}
            The side to store the loss for
        loss: list
            The list of loss ``floats`` for this side
        """
        if not self._tensorboard:
    def _log_tensorboard(self, side, loss):
        """ Log current loss to Tensorboard log files

        Parameters
        ----------
        side: {"a", "b"}
            The side to store the loss for
        loss: list
            The list of loss ``floats`` for this side
        """
        if not self._tensorboard:
    def _log_tensorboard(self, side, loss):
        """ Log current loss to Tensorboard log files

        Parameters
        ----------
        side: {"a", "b"}
            The side to store the loss for
        loss: list
            The list of loss ``floats`` for this side
        """
        if not self._tensorboard:
              return
          logger.trace("Updating TensorBoard log: '%s'", side)
          logs = {log[0]: log[1]
                  for log in zip(self.model.state.loss_names[side], loss)}
          self.tensorboard[side].on_batch_end(self.model.state.iterations, logs)
keep keep keep keep replace replace replace keep keep keep keep keep
 Smart Masks - Training Implementation (#914)

* Smart Masks - Training

- Reinstate smart mask training code
- Reinstate mask_type back to model.config
- change 'replicate_input_mask to 'learn_mask'
- Add learn mask option
- Add mask loading from alignments to plugins.train.trainer
- Add mask_blur and mask threshold options
- _base.py - Pass mask options through training_opts dict
- plugins.train.model - check for mask_type not None for learn_mask and penalized_mask_loss
- Limit alignments loading to just those faces that appear in the training folder
- Raise error if not all training images have an alignment, and alignment file is required
- lib.training_data - Mask generation code
- lib.faces_detect - cv2 dimension stripping bugfix
- Remove cv2 linting code

* Update mask helptext in cli.py

* Fix Warp to Landmarks
Remove SHA1 hashing from training data

* Update mask training config

* Capture missing masks at training init

* lib.image.read_image_batch - Return filenames with batch for ordering

* scripts.train - Documentation

* plugins.train.trainer - documentation

* Ensure backward compatibility.
Fix convert for new predicted masks

* Update removed masks to components for legacy models. 
plugins/train/trainer/_base.py