                         (self._model.iterations - 1) % snapshot_interval == 0)
  
          model_inputs, model_targets = self._feeder.get_batch()
          try:
              loss = self._model.model.train_on_batch(model_inputs, y=model_targets)
          except tf_errors.ResourceExhaustedError as err:
              msg = ("You do not have enough GPU memory available to train the selected model at "
                     "the selected settings. You can try a number of things:"
                     "\n1) Close any other application that is using your GPU (web browsers are "
                     "particularly bad for this)."
 
</s> remove         logger.trace("Training one step: (iteration: %s)", self._model.iterations)
        do_preview = viewer is not None
 </s> add         logger.trace("Training one step: (iteration: %s)", self._model.iterations)  # type: ignore </s> remove         Returns
        -------
        bool
            ``True`` if the cache has been reset because of a face centering change due to
            legacy alignments, otherwise ``False``. """
        retval = self._has_reset
        if retval:
            logger.debug("Resetting 'has_reset' flag")
            self._has_reset = False
        return retval
 </s> add         Raises
        ------
        :class:`FaceswapError`
            If the number of images loaded is smaller than the selected batch size
        """
        length = len(self._images)
        msg = ("Number of images is lower than batch-size (Note that too few images may lead to "
               f"bad training). # images: {length}, batch-size: {self._batch_size}")
        try:
            assert length >= self._batch_size, msg
        except AssertionError as err:
            msg += ("\nYou should increase the number of images in your training set or lower "
                    "your batch-size.")
            raise FaceswapError(msg) from err </s> remove             # Ensure tensorflow doesn't pin all threads to one core when using Math Kernel Library
            os.environ["TF_MIN_GPU_MULTIPROCESSOR_COUNT"] = "4"
            os.environ["KMP_AFFINITY"] = "disabled"
 </s> add  </s> remove         logger.debug("Queue batches: (image_count: %s, batchsize: %s, side: '%s', do_shuffle: %s, "
                     "is_preview, %s, is_timelapse: %s)", len(images), batchsize, side, do_shuffle,
                     is_preview, is_timelapse)
        self._batchsize = batchsize
        self._face_cache = _get_cache(side, images, self._config)
        self._processing = ImageAugmentation(batchsize,
                                             is_preview or is_timelapse,
                                             self._model_input_size,
                                             self._model_output_shapes,
                                             self._coverage_ratio,
                                             self._config)

        if self._warp_to_landmarks and not self._face_cache.partially_loaded:
            self._face_cache.pre_fill(images, side)

        args = (images, side, do_shuffle, batchsize)
        batcher = BackgroundGenerator(self._minibatch, thread_count=2, args=args)
        return batcher.iterator()

    # << INTERNAL METHODS >> #
    def _validate_samples(self, data):
        """ Ensures that the total number of images within :attr:`images` is greater or equal to
        the selected :attr:`batchsize`. Raises an exception if this is not the case. """
        length = len(data)
        msg = ("Number of images is lower than batch-size (Note that too few "
               "images may lead to bad training). # images: {}, "
               "batch-size: {}".format(length, self._batchsize))
        try:
            assert length >= self._batchsize, msg
        except AssertionError as err:
            msg += ("\nYou should increase the number of images in your training set or lower "
                    "your batch-size.")
            raise FaceswapError(msg) from err

    def _minibatch(self, images, side, do_shuffle, batchsize):
        """ A generator function that yields the augmented, target and sample images.
        see :func:`minibatch_ab` for more details on the output. """
        logger.debug("Loading minibatch generator: (image_count: %s, side: '%s', do_shuffle: %s)",
                     len(images), side, do_shuffle)
        self._validate_samples(images)

        def _img_iter(imgs):
            while True:
                if do_shuffle:
                    shuffle(imgs)
                for img in imgs:
                    yield img

        img_iter = _img_iter(images)
        while True:
            img_paths = [next(img_iter) for _ in range(batchsize)]
            yield self._process_batch(img_paths, side)

        logger.debug("Finished minibatch generator: (side: '%s')", side)
 </s> add         logger.trace("Compiling targets: batch shape: %s", batch.shape)  # type: ignore
        if len(self._output_sizes) == 1 and self._output_sizes[0] == self._process_size:
            # Rolling buffer here makes next to no difference, so just create array on the fly
            retval = [self._to_float32(batch)]
        else:
            retval = [self._to_float32(np.array([cv2.resize(image, (size, size), cv2.INTER_AREA)
                                                 for image in batch]))
                      for size in self._output_sizes]
        logger.trace("Processed targets: %s", [t.shape for t in retval])  # type: ignore
        return retval
            loss: List[float] = self._model.model.train_on_batch(model_inputs, y=model_targets)
                         (self._model.iterations - 1) % snapshot_interval == 0)
  
          model_inputs, model_targets = self._feeder.get_batch()
          try:
            loss: List[float] = self._model.model.train_on_batch(model_inputs, y=model_targets)
          except tf_errors.ResourceExhaustedError as err:
              msg = ("You do not have enough GPU memory available to train the selected model at "
                     "the selected settings. You can try a number of things:"
                     "\n1) Close any other application that is using your GPU (web browsers are "
                     "particularly bad for this)."
keep keep keep keep replace keep keep keep keep keep
 Data Augmentation update (#1263)

- lib.detected_face
    - Subclass Masks for Landmark based masks
    - Add training mask propery + methods to DetectedFace
  - lib.training_training
    - subclass TrainingDataGenerator for training and preview data
    - Split cache into own module
    - Reduce thread count to 1 to prevent image corruption + data re-use
    - Process on largest model input/output size rather than stored image size
    - Size and crop masks during caching stage
    - Implement ring buffer for data flow
    - Fix preview reload bug
  - augmentation
    - typing
    - switch color aug order
    - better initialization
    - Fix warp + landmark warp to correctly apply at different image scales
    - Slightly improved warp caching
    - Don't store whether image is_preview. Handle all data as training images implicitly
  - plugins.trainer: Typing and fixes to work with trainingdata refactor 
plugins/train/trainer/_base.py