          tensorboard = dict()
  
          for side in self.sides:
              logger.debug("Setting up TensorBoard Logging. Side: %s", side)
              log_dir = os.path.join(str(self.model.model_dir),
                                     "{}_logs".format(self.model.name),
                                     side,
                                     "session_{}".format(self.model.state.session_id))
              tbs = tf.keras.callbacks.TensorBoard(log_dir=log_dir, **self.tensorboard_kwargs)
              tbs.set_model(self.model.predictors[side])
              tensorboard[side] = tbs
 
</s> remove                                    "session_{}".format(self.model.state.session_id))
            tbs = tf.keras.callbacks.TensorBoard(log_dir=log_dir, **self.tensorboard_kwargs)
            tbs.set_model(self.model.predictors[side])
 </s> add                                    "session_{}".format(self._model.state.session_id))
            tbs = tf.keras.callbacks.TensorBoard(log_dir=log_dir, **self._tensorboard_kwargs)
            tbs.set_model(self._model.predictors[side]) </s> remove         for side in self.sides:
 </s> add         for side in self._sides: </s> remove         """ Perform timelapse """
        logger.debug("Getting timelapse samples: '%s'", side)
        if not self.output_file:
            self.setup(**timelapse_kwargs)
        self.samples.images[side] = self.batchers[side].compile_timelapse_sample()
        logger.debug("Got timelapse samples: '%s' - %s", side, len(self.samples.images[side]))

    def setup(self, input_a=None, input_b=None, output=None):
        """ Set the timelapse output folder """
        logger.debug("Setting up timelapse")
 </s> add         """ Compile the time-lapse preview

        Parameters
        ----------
        side: {"a" or "b"}
            The side that the time-lapse is being generated for
        timelapse_kwargs: dict
            The keyword arguments for setting up the time-lapse. All values should be full paths
            the keys being `input_a`, `input_b`, `output`
        """
        logger.debug("Getting time-lapse samples: '%s'", side)
        if not self._output_file:
            self._setup(**timelapse_kwargs)
        self._samples.images[side] = self._batchers[side].compile_timelapse_sample()
        logger.debug("Got time-lapse samples: '%s' - %s", side, len(self._samples.images[side]))

    def _setup(self, input_a=None, input_b=None, output=None):
        """ Setup the time-lapse folder locations and the time-lapse feed.

        Parameters
        ----------
        input_a: str
            The full path to the time-lapse input folder containing faces for the "a" side
        input_b: str
            The full path to the time-lapse input folder containing faces for the "b" side
        output: str, optional
            The full path to the time-lapse output folder. If ``None`` is provided this will
            default to the model folder
        """
        logger.debug("Setting up time-lapse") </s> remove             landmarks[side] = self.transform_landmarks(alignments)
        return landmarks
 </s> add             retval[side] = self._to_detected_faces(alignments, side)
        logger.debug("Returning: %s", {k: len(v) for k, v in retval.items()})
        return retval </s> remove         for side, tensorboard in self.tensorboard.items():
 </s> add         for side, tensorboard in self._tensorboard.items():
            log_dir = os.path.join(str(self._model.model_dir),
                                   "{}_logs".format(self._model.name),
          tensorboard = dict()
  
          for side in self.sides:
              logger.debug("Setting up TensorBoard Logging. Side: %s", side)
            log_dir = os.path.join(str(self._model.model_dir),
                                   "{}_logs".format(self._model.name),
            log_dir = os.path.join(str(self._model.model_dir),
                                   "{}_logs".format(self._model.name),
                                     side,
                                     "session_{}".format(self.model.state.session_id))
              tbs = tf.keras.callbacks.TensorBoard(log_dir=log_dir, **self.tensorboard_kwargs)
              tbs.set_model(self.model.predictors[side])
              tensorboard[side] = tbs
keep keep keep keep replace replace keep keep keep keep keep
 Smart Masks - Training Implementation (#914)

* Smart Masks - Training

- Reinstate smart mask training code
- Reinstate mask_type back to model.config
- change 'replicate_input_mask to 'learn_mask'
- Add learn mask option
- Add mask loading from alignments to plugins.train.trainer
- Add mask_blur and mask threshold options
- _base.py - Pass mask options through training_opts dict
- plugins.train.model - check for mask_type not None for learn_mask and penalized_mask_loss
- Limit alignments loading to just those faces that appear in the training folder
- Raise error if not all training images have an alignment, and alignment file is required
- lib.training_data - Mask generation code
- lib.faces_detect - cv2 dimension stripping bugfix
- Remove cv2 linting code

* Update mask helptext in cli.py

* Fix Warp to Landmarks
Remove SHA1 hashing from training data

* Update mask training config

* Capture missing masks at training init

* lib.image.read_image_batch - Return filenames with batch for ordering

* scripts.train - Documentation

* plugins.train.trainer - documentation

* Ensure backward compatibility.
Fix convert for new predicted masks

* Update removed masks to components for legacy models. 
plugins/train/trainer/_base.py