          else:
              file_list = [path for path in get_image_paths(input_aligned_dir)]
              logger.info("Getting Face Hashes for selected Aligned Images")
              for face in tqdm(file_list, desc="Hashing Faces"):
                  face_hashes.append(hash_image_file(face))
              logger.debug("Face Hashes: %s", (len(face_hashes)))
              if not face_hashes:
                  raise FaceswapError("Aligned directory is empty, no faces will be converted!")
              if len(face_hashes) <= len(self.input_images) / 3:
                  logger.warning("Aligned directory contains far fewer images than the input "
 
</s> remove         except subprocess.CalledProcessError as err:
            out = err.output.decode(errors="ignore")
            raise RuntimeError("FFMEG call failed with {}:\n{}".format(err.returncode, out))
        except subprocess.TimeoutExpired as err:
            this_attempt = attempt + 1
            if this_attempt == attempts:
                msg = ("FFMPEG hung while attempting to obtain the frame count. "
                       "Sometimes this issue resolves itself, so you can try running again. "
                       "Otherwise use the Effmpeg Tool to extract the frames from your video into "
                       "a folder, and then run the requested Faceswap process on that folder.")
                raise FaceswapError(msg) from err
            logger.warning("FFMPEG hung while attempting to obtain the frame count. "
                           "Retrying %s of %s", this_attempt + 1, attempts)
            continue

    # Note that other than with the subprocess calls below, ffmpeg wont hang here.
    # Worst case Python will stop/crash and ffmpeg will continue running until done.

    nframes = nsecs = None
    for line in reversed(out.splitlines()):
        if not line.startswith(b"frame="):
            continue
        line = line.decode(errors="ignore")
        logger.debug("frame line: '%s'", line)
        idx = line.find("frame=")
        if idx >= 0:
            splitframes = line[idx:].split("=", 1)[-1].lstrip().split(" ", 1)[0].strip()
            nframes = int(splitframes)
        idx = line.find("time=")
        if idx >= 0:
            splittime = line[idx:].split("=", 1)[-1].lstrip().split(" ", 1)[0].strip()
            nsecs = convert_to_secs(*splittime.split(":"))
        logger.debug("nframes: %s, nsecs: %s", nframes, nsecs)
        return nframes, nsecs

    raise RuntimeError("Could not get number of frames")  # pragma: no cover
 </s> add         elif parts[1] == path:  # sentinel for relative paths
            allparts.insert(0, parts[1])
            break
        else:
            path = parts[0]
            allparts.insert(0, parts[1])
    logger.trace("path: %s, allparts: %s", path, allparts)
    return allparts </s> remove     def validate_samples(self, data):
        """ Check the total number of images against batchsize and return
            the total number of images """
 </s> add     # << INTERNAL METHODS >> #
    def _set__mask_class(self):
        """ Returns the correct mask class from :mod:`lib`.model.masks` as defined in the
        :attr:`mask_type` parameter. """
        mask_type = self._training_opts.get("mask_type", None)
        if mask_type:
            logger.debug("Mask type: '%s'", mask_type)
            _mask_class = getattr(masks, mask_type)
        else:
            _mask_class = None
        logger.debug("Mask class: %s", _mask_class)
        return _mask_class

    def _validate_samples(self, data):
        """ Ensures that the total number of images within :attr:`images` is greater or equal to
        the selected :attr:`batchsize`. Raises an exception if this is not the case. """ </s> remove     def get_landmarks(self, filename, image, side):
        """ Return the landmarks for this face """
        logger.trace("Retrieving landmarks: (filename: '%s', side: '%s'", filename, side)
        lm_key = sha1(image).hexdigest()
        try:
            src_points = self.landmarks[side][lm_key]
        except KeyError as err:
            msg = ("At least one of your images does not have a matching entry in your alignments "
                   "file."
 </s> add     def _get_landmarks(self, filenames, batch, side):
        """ Obtains the 68 Point Landmarks for the images in this batch. This is only called if
        config item ``warp_to_landmarks`` is ``True`` or if :attr:`mask_type` is not ``None``. If
        the landmarks for an image cannot be found, then an error is raised. """
        logger.trace("Retrieving landmarks: (filenames: '%s', side: '%s'", filenames, side)
        src_points = [self._landmarks[side].get(sha1(face).hexdigest(), None) for face in batch]

        # Raise error on missing alignments
        if not all(isinstance(pts, np.ndarray) for pts in src_points):
            indices = [idx for idx, hsh in enumerate(src_points) if hsh is None]
            missing = [filenames[idx] for idx in indices]
            msg = ("Files missing alignments for this batch: {}"
                   "\nAt least one of your images does not have a matching entry in your "
                   "alignments file." </s> remove                    "\nThe specific file that caused the failure was '{}' which has a hash of {}."
                   "\nMost likely there will be more than just this file missing from the "
 </s> add                    "\nThe specific files that caused this failure are listed above."
                   "\nMost likely there will be more than just these files missing from the " </s> remove     """ Generate training data for models """
 </s> add     """ A Training Data Generator for compiling data for feeding to a model.

    This class is called from :mod:`plugins.train.trainer._base` and launches a background
    iterator that compiles augmented data, target data and sample data.

    Parameters
    ----------
    model_input_size: int
        The expected input size for the model. It is assumed that the input to the model is always
        a square image. This is the size, in pixels, of the `width` and the `height` of the input
        to the model.
    model_output_shapes: list
        A list of tuples defining the output shapes from the model, in the order that the outputs
        are returned. The tuples should be in (`height`, `width`, `channels`) format.
    training_opts: dict
        This is a dictionary of model training options as defined in
        :mod:`plugins.train.model._base`. These options will be defined by the user from the
        provided cli options or from the model ``config.ini``. At a minimum this ``dict`` should
        contain the following keys:

        * **coverage_ratio** (`float`) - The ratio of the training image to be trained on. \
        Dictates how much of the image will be cropped out. Eg: a coverage ratio of 0.625 \
        will result in cropping a 160px box from a 256px image (256 * 0.625 = 160).

        * **augment_color** (`bool`) - ``True`` if color is to be augmented, otherwise ``False`` \

        * **no_flip** (`bool`) - ``True`` if the image shouldn't be randomly flipped as part of \
        augmentation, otherwise ``False``

        * **mask_type** (`str`) - The mask type to be used (as defined in \
        :mod:`lib.model.masks`). If not ``None`` then the additional key ``landmarks`` must be \
        provided.

        * **warp_to_landmarks** (`bool`) - ``True`` if the random warp method should warp to \
        similar landmarks from the other side, ``False`` if the standard random warp method \
        should be used. If ``True`` then the additional key ``landmarks`` must be provided.

        * **landmarks** (`numpy.ndarray`, `optional`). Required if using a :attr:`mask_type` is \
        not ``None`` or :attr:`warp_to_landmarks` is ``True``. The 68 point face landmarks from \
        an alignments file.

    config: dict
        The configuration ``dict`` generated from :file:`config.train.ini` containing the trainer \
        plugin configuration options.
    """
                face_hashes.append(read_image_hash(face))
          else:
              file_list = [path for path in get_image_paths(input_aligned_dir)]
              logger.info("Getting Face Hashes for selected Aligned Images")
              for face in tqdm(file_list, desc="Hashing Faces"):
                face_hashes.append(read_image_hash(face))
              logger.debug("Face Hashes: %s", (len(face_hashes)))
              if not face_hashes:
                  raise FaceswapError("Aligned directory is empty, no faces will be converted!")
              if len(face_hashes) <= len(self.input_images) / 3:
                  logger.warning("Aligned directory contains far fewer images than the input "
keep keep keep keep replace keep keep keep keep keep
 Optimize Data Augmentation (#881)

* Move image utils to lib.image
* Add .pylintrc file
* Remove some cv2 pylint ignores
* TrainingData: Load images from disk in batches
* TrainingData: get_landmarks to batch
* TrainingData: transform and flip to batches
* TrainingData: Optimize color augmentation
* TrainingData: Optimize target and random_warp
* TrainingData - Convert _get_closest_match for batching
* TrainingData: Warp To Landmarks optimized
* Save models to threadpoolexecutor
* Move stack_images, Rename ImageManipulation. ImageAugmentation Docstrings
* Masks: Set dtype and threshold for lib.masks based on input face
* Docstrings and Documentation 
scripts/convert.py