          logger.info("Enabled TensorBoard Logging")
          return tensorboard
  
      @property
      def tensorboard_kwargs(self):
          """ TF 1.13 + needs an additional kwarg which is not valid for earlier versions """
          kwargs = dict(histogram_freq=0,  # Must be 0 or hangs
                        batch_size=64,
                        write_graph=True,
                        write_grads=True)
          tf_version = [int(ver) for ver in tf.__version__.split(".") if ver.isdigit()]
 
</s> remove                                    "session_{}".format(self.model.state.session_id))
            tbs = tf.keras.callbacks.TensorBoard(log_dir=log_dir, **self.tensorboard_kwargs)
            tbs.set_model(self.model.predictors[side])
 </s> add                                    "session_{}".format(self._model.state.session_id))
            tbs = tf.keras.callbacks.TensorBoard(log_dir=log_dir, **self._tensorboard_kwargs)
            tbs.set_model(self._model.predictors[side]) </s> remove         for side in self.sides:
 </s> add         for side in self._sides: </s> remove         """ Indicate training end to Tensorboard """
        if not self.tensorboard:
 </s> add         """ Stop Tensorboard logging.

        Tensorboard logging needs to be explicitly shutdown on training termination. Called from
        :class:`scripts.train.Train` when training is stopped.
         """
        if not self._tensorboard: </s> remove         if self.pingpong.active:
 </s> add         if self._pingpong.active: </s> remove         config item ``warp_to_landmarks`` is ``True`` or if :attr:`mask_type` is not ``None``. If
        the landmarks for an image cannot be found, then an error is raised. """
 </s> add         config item ``warp_to_landmarks`` is ``True``. If the landmarks for an image cannot be
        found, then an error is raised. """
    def _tensorboard_kwargs(self):
        """ dict: The keyword arguments to be passed to :class:`tf.keras.callbacks.TensorBoard`.
        NB: Tensorflow 1.13 + needs an additional keyword argument which is not valid for earlier
        versions """
          logger.info("Enabled TensorBoard Logging")
          return tensorboard
  
      @property
    def _tensorboard_kwargs(self):
        """ dict: The keyword arguments to be passed to :class:`tf.keras.callbacks.TensorBoard`.
        NB: Tensorflow 1.13 + needs an additional keyword argument which is not valid for earlier
        versions """
    def _tensorboard_kwargs(self):
        """ dict: The keyword arguments to be passed to :class:`tf.keras.callbacks.TensorBoard`.
        NB: Tensorflow 1.13 + needs an additional keyword argument which is not valid for earlier
        versions """
          kwargs = dict(histogram_freq=0,  # Must be 0 or hangs
                        batch_size=64,
                        write_graph=True,
                        write_grads=True)
          tf_version = [int(ver) for ver in tf.__version__.split(".") if ver.isdigit()]
keep keep keep keep replace replace keep keep keep keep keep
 Smart Masks - Training Implementation (#914)

* Smart Masks - Training

- Reinstate smart mask training code
- Reinstate mask_type back to model.config
- change 'replicate_input_mask to 'learn_mask'
- Add learn mask option
- Add mask loading from alignments to plugins.train.trainer
- Add mask_blur and mask threshold options
- _base.py - Pass mask options through training_opts dict
- plugins.train.model - check for mask_type not None for learn_mask and penalized_mask_loss
- Limit alignments loading to just those faces that appear in the training folder
- Raise error if not all training images have an alignment, and alignment file is required
- lib.training_data - Mask generation code
- lib.faces_detect - cv2 dimension stripping bugfix
- Remove cv2 linting code

* Update mask helptext in cli.py

* Fix Warp to Landmarks
Remove SHA1 hashing from training data

* Update mask training config

* Capture missing masks at training init

* lib.image.read_image_batch - Return filenames with batch for ordering

* scripts.train - Documentation

* plugins.train.trainer - documentation

* Ensure backward compatibility.
Fix convert for new predicted masks

* Update removed masks to components for legacy models. 
plugins/train/trainer/_base.py