          if self.landmarks_required:
              landmarks = Landmarks(self.model.training_opts).landmarks
              self.model.training_opts["landmarks"] = landmarks
  
      def set_tensorboard(self):
          """ Set up tensorboard callback """
          if self.model.training_opts["no_logs"]:
              logger.verbose("TensorBoard logging disabled")
              return None
          if self.pingpong.active:
              # Currently TensorBoard uses the tf.session, meaning that VRAM does not
              # get cleared when model switching
 
</s> remove         if self.pingpong.active:
 </s> add         if self._pingpong.active: </s> remove     def process_training_opts(self):
        """ Override for processing model specific training options """
        logger.debug(self.model.training_opts)
        if self.landmarks_required:
            landmarks = Landmarks(self.model.training_opts).landmarks
            self.model.training_opts["landmarks"] = landmarks
 </s> add     def _process_training_opts(self):
        """ Extrapolate alignments and masks from the alignments file into
        :attr:`_model.training_opts`."""
        logger.debug(self._model.training_opts)
        if not self._landmarks_required and not self._use_mask:
            return </s> remove         """ Indicate training end to Tensorboard """
        if not self.tensorboard:
 </s> add         """ Stop Tensorboard logging.

        Tensorboard logging needs to be explicitly shutdown on training termination. Called from
        :class:`scripts.train.Train` when training is stopped.
         """
        if not self._tensorboard: </s> remove         """ Set the timelapse dictionary """
        logger.debug("Ouputting timelapse")
        image = self.samples.show_sample()
 </s> add         """ Write the created time-lapse to the specified output folder. """
        logger.debug("Ouputting time-lapse")
        image = self._samples.show_sample() </s> remove     def transform_landmarks(self, alignments):
        """ For each face transform landmarks and return """
        landmarks = dict()
        for _, faces, _, _ in alignments.yield_faces():
            for face in faces:
 </s> add     def _to_detected_faces(self, alignments, side):
        """ Convert alignments to DetectedFace objects.

        Filter the detected faces to only those that exist in the training folders.

        Parameters
        ----------
        alignments: :class:`lib.alignments.Alignments`
            The alignments for the current faces
        side: {"a" or "b"}
            The side being processed

        Returns
        -------
        list
            List of :class:`lib.faces_detect.DetectedFace` objects
        """
        skip_count = 0
        side_hashes = set(self._hashes[side]["hashes"])
        detected_faces = []
        for _, faces, _, filename in alignments.yield_faces():
            for idx, face in enumerate(faces):
                if not self._validate_face(face, filename, idx, side, side_hashes):
                    skip_count += 1
                    continue
        alignments = TrainingAlignments(self._model.training_opts, self._images)
        if self._landmarks_required:
            logger.debug("Adding landmarks to training opts dict")
            self._model.training_opts["landmarks"] = alignments.landmarks

        if self._use_mask:
            logger.debug("Adding masks to training opts dict")
            self._model.training_opts["masks"] = alignments.masks

    def _set_tensorboard(self):
        """ Set up Tensorboard callback for logging loss.

        Bypassed if command line option "no-logs" has been selected.

        Returns
        -------
        dict:
            2 Dictionary keys of "a" and "b" the values of which are the
        :class:`tf.keras.callbacks.TensorBoard` objects for the respective sides.
        """
        if self._model.training_opts["no_logs"]:
          if self.landmarks_required:
              landmarks = Landmarks(self.model.training_opts).landmarks
              self.model.training_opts["landmarks"] = landmarks
  
        alignments = TrainingAlignments(self._model.training_opts, self._images)
        if self._landmarks_required:
            logger.debug("Adding landmarks to training opts dict")
            self._model.training_opts["landmarks"] = alignments.landmarks

        if self._use_mask:
            logger.debug("Adding masks to training opts dict")
            self._model.training_opts["masks"] = alignments.masks

    def _set_tensorboard(self):
        """ Set up Tensorboard callback for logging loss.

        Bypassed if command line option "no-logs" has been selected.

        Returns
        -------
        dict:
            2 Dictionary keys of "a" and "b" the values of which are the
        :class:`tf.keras.callbacks.TensorBoard` objects for the respective sides.
        """
        if self._model.training_opts["no_logs"]:
        alignments = TrainingAlignments(self._model.training_opts, self._images)
        if self._landmarks_required:
            logger.debug("Adding landmarks to training opts dict")
            self._model.training_opts["landmarks"] = alignments.landmarks

        if self._use_mask:
            logger.debug("Adding masks to training opts dict")
            self._model.training_opts["masks"] = alignments.masks

    def _set_tensorboard(self):
        """ Set up Tensorboard callback for logging loss.

        Bypassed if command line option "no-logs" has been selected.

        Returns
        -------
        dict:
            2 Dictionary keys of "a" and "b" the values of which are the
        :class:`tf.keras.callbacks.TensorBoard` objects for the respective sides.
        """
        if self._model.training_opts["no_logs"]:
        alignments = TrainingAlignments(self._model.training_opts, self._images)
        if self._landmarks_required:
            logger.debug("Adding landmarks to training opts dict")
            self._model.training_opts["landmarks"] = alignments.landmarks

        if self._use_mask:
            logger.debug("Adding masks to training opts dict")
            self._model.training_opts["masks"] = alignments.masks

    def _set_tensorboard(self):
        """ Set up Tensorboard callback for logging loss.

        Bypassed if command line option "no-logs" has been selected.

        Returns
        -------
        dict:
            2 Dictionary keys of "a" and "b" the values of which are the
        :class:`tf.keras.callbacks.TensorBoard` objects for the respective sides.
        """
        if self._model.training_opts["no_logs"]:
              logger.verbose("TensorBoard logging disabled")
              return None
          if self.pingpong.active:
              # Currently TensorBoard uses the tf.session, meaning that VRAM does not
              # get cleared when model switching
keep keep keep keep replace replace replace keep keep keep keep keep
 Smart Masks - Training Implementation (#914)

* Smart Masks - Training

- Reinstate smart mask training code
- Reinstate mask_type back to model.config
- change 'replicate_input_mask to 'learn_mask'
- Add learn mask option
- Add mask loading from alignments to plugins.train.trainer
- Add mask_blur and mask threshold options
- _base.py - Pass mask options through training_opts dict
- plugins.train.model - check for mask_type not None for learn_mask and penalized_mask_loss
- Limit alignments loading to just those faces that appear in the training folder
- Raise error if not all training images have an alignment, and alignment file is required
- lib.training_data - Mask generation code
- lib.faces_detect - cv2 dimension stripping bugfix
- Remove cv2 linting code

* Update mask helptext in cli.py

* Fix Warp to Landmarks
Remove SHA1 hashing from training data

* Update mask training config

* Capture missing masks at training init

* lib.image.read_image_batch - Return filenames with batch for ordering

* scripts.train - Documentation

* plugins.train.trainer - documentation

* Ensure backward compatibility.
Fix convert for new predicted masks

* Update removed masks to components for legacy models. 
plugins/train/trainer/_base.py