  
  class Batcher():
      """ Batch images from a single side """
      def __init__(self, side, images, model, use_mask, batch_size, config):
          logger.debug("Initializing %s: side: '%s', num_images: %s, batch_size: %s, config: %s)",
                       self.__class__.__name__, side, len(images), batch_size, config)
          self.model = model
          self.use_mask = use_mask
          self.side = side
          self.images = images
          self.config = config
          self.target = None
          self.samples = None
          self.mask = None
  
          generator = self.load_generator()
          self.feed = generator.minibatch_ab(images, batch_size, self.side)
  
          self.preview_feed = None
          self.timelapse_feed = None
  
      def load_generator(self):
          """ Pass arguments to TrainingDataGenerator and return object """
          logger.debug("Loading generator: %s", self.side)
          input_size = self.model.input_shape[0]
          output_shapes = self.model.output_shapes
          logger.debug("input_size: %s, output_shapes: %s", input_size, output_shapes)
          generator = TrainingDataGenerator(input_size,
                                            output_shapes,
                                            self.model.training_opts,
                                            self.config)
 
</s> remove                                           self.model.training_opts,
                                          self.config)
 </s> add                                           self._model.training_opts,
                                          self._config) </s> remove     """ Batch images from a single side """
 </s> add     """ Handles the processing of a Batch for a single side.

    Parameters
    ----------
    side: {"a" or "b"}
        The side that this :class:`Batcher` belongs to
    images: list
        The list of full paths to the training images for this :class:`Batcher`
    model: plugin from :mod:`plugins.train.model`
        The selected model that will be running this trainer
    use_mask: bool
        ``True`` if a mask is required for training otherwise ``False``
    batch_size: int
        The size of the batch to be processed at each iteration
    config: :class:`lib.config.FaceswapConfig`
        The configuration for this trainer
    """ </s> remove         self.model = model
        self.use_mask = use_mask
 </s> add         self._model = model
        self._use_mask = use_mask </s> remove                      "preview_images: %s, batchers: '%s')", self.__class__.__name__, model,
                     use_mask, coverage_ratio, preview_images, batchers)
        self.preview_images = preview_images
        self.samples = Samples(model, use_mask, coverage_ratio)
        self.model = model
        self.batchers = batchers
        self.output_file = None
 </s> add                      "image_count: %s, batchers: '%s')", self.__class__.__name__, model,
                     use_mask, coverage_ratio, image_count, batchers)
        self._num_images = image_count
        self._samples = Samples(model, use_mask, coverage_ratio)
        self._model = model
        self._batchers = batchers
        self._output_file = None </s> remove         self.config = get_config(".".join(self.__module__.split(".")[-2:]), configfile=configfile)
        self.batch_size = batch_size
        self.model = model
        self.model.state.add_session_batchsize(batch_size)
        self.images = images
        self.sides = sorted(key for key in self.images.keys())

        self.process_training_opts()
        self.pingpong = PingPong(model, self.sides)

        self.batchers = {side: Batcher(side,
                                       images[side],
                                       self.model,
                                       self.use_mask,
                                       batch_size,
                                       self.config)
                         for side in self.sides}

        self.tensorboard = self.set_tensorboard()
        self.samples = Samples(self.model,
                               self.use_mask,
                               self.model.training_opts["coverage_ratio"],
                               self.model.training_opts["preview_scaling"])
        self.timelapse = Timelapse(self.model,
                                   self.use_mask,
                                   self.model.training_opts["coverage_ratio"],
                                   self.config.get("preview_images", 14),
                                   self.batchers)
 </s> add         self._config = _get_config(".".join(self.__module__.split(".")[-2:]),
                                   configfile=configfile)
        self._model = model
        self._model.state.add_session_batchsize(batch_size)
        self._images = images
        self._sides = sorted(key for key in self._images.keys())

        self._process_training_opts()
        self._pingpong = PingPong(model, self._sides)

        self._batchers = {side: Batcher(side,
                                        images[side],
                                        self._model,
                                        self._use_mask,
                                        batch_size,
                                        self._config)
                          for side in self._sides}

        self._tensorboard = self._set_tensorboard()
        self._samples = Samples(self._model,
                                self._use_mask,
                                self._model.training_opts["coverage_ratio"],
                                self._model.training_opts["preview_scaling"])
        self._timelapse = Timelapse(self._model,
                                    self._use_mask,
                                    self._model.training_opts["coverage_ratio"],
                                    self._config.get("preview_images", 14),
                                    self._batchers)
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
  
  class Batcher():
      """ Batch images from a single side """
      def __init__(self, side, images, model, use_mask, batch_size, config):
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
        logger.debug("Initializing %s: side: '%s', num_images: %s, use_mask: %s, batch_size: %s, "
                     "config: %s)",
                     self.__class__.__name__, side, len(images), use_mask, batch_size, config)
        self._model = model
        self._use_mask = use_mask
        self._side = side
        self._images = images
        self._config = config
        self._target = None
        self._samples = None
        self._masks = None

        generator = self._load_generator()
        self._feed = generator.minibatch_ab(images, batch_size, self._side)

        self._preview_feed = None
        self._timelapse_feed = None
        self._set_preview_feed()

    def _load_generator(self):
        """ Load the :class:`lib.training_data.TrainingDataGenerator` for this batcher """
        logger.debug("Loading generator: %s", self._side)
        input_size = self._model.input_shape[0]
        output_shapes = self._model.output_shapes
          logger.debug("input_size: %s, output_shapes: %s", input_size, output_shapes)
          generator = TrainingDataGenerator(input_size,
                                            output_shapes,
                                            self.model.training_opts,
                                            self.config)
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 Smart Masks - Training Implementation (#914)

* Smart Masks - Training

- Reinstate smart mask training code
- Reinstate mask_type back to model.config
- change 'replicate_input_mask to 'learn_mask'
- Add learn mask option
- Add mask loading from alignments to plugins.train.trainer
- Add mask_blur and mask threshold options
- _base.py - Pass mask options through training_opts dict
- plugins.train.model - check for mask_type not None for learn_mask and penalized_mask_loss
- Limit alignments loading to just those faces that appear in the training folder
- Raise error if not all training images have an alignment, and alignment file is required
- lib.training_data - Mask generation code
- lib.faces_detect - cv2 dimension stripping bugfix
- Remove cv2 linting code

* Update mask helptext in cli.py

* Fix Warp to Landmarks
Remove SHA1 hashing from training data

* Update mask training config

* Capture missing masks at training init

* lib.image.read_image_batch - Return filenames with batch for ordering

* scripts.train - Documentation

* plugins.train.trainer - documentation

* Ensure backward compatibility.
Fix convert for new predicted masks

* Update removed masks to components for legacy models. 
plugins/train/trainer/_base.py