      def load_disk_frames(self):
          """ Load frames from disk """
          logger.debug("Input is separate Frames. Loading images")
          for filename in self.input_images:
              image = cv2_read_img(filename, raise_error=False)
              if image is None:
                  continue
              yield filename, image
  
      def load_video_frames(self):
 
</s> remove def full_path_split(path):
    """ Split a given path into all of it's separate components """
    logger = logging.getLogger(__name__)  # pylint:disable=invalid-name
    allparts = list()
    while True:
        parts = os.path.split(path)
        if parts[0] == path:   # sentinel for absolute paths
            allparts.insert(0, parts[0])
            break
        elif parts[1] == path:  # sentinel for relative paths
            allparts.insert(0, parts[1])
            break
        else:
            path = parts[0]
            allparts.insert(0, parts[1])
    logger.trace("path: %s, allparts: %s", path, allparts)
    return allparts


def cv2_read_img(filename, raise_error=False):
    """ Read an image with cv2 and check that an image was actually loaded.
        Logs an error if the image returned is None. or an error has occured.

        Pass raise_error=True if error should be raised """
    logger = logging.getLogger(__name__)  # pylint:disable=invalid-name
    logger.trace("Requested image: '%s'", filename)
    success = True
    image = None
    try:
        image = cv2.imread(filename)  # pylint:disable=no-member,c-extension-no-member
        if image is None:
            raise ValueError
    except TypeError:
        success = False
        msg = "Error while reading image (TypeError): '{}'".format(filename)
        logger.error(msg)
        if raise_error:
            raise Exception(msg)
    except ValueError:
        success = False
        msg = ("Error while reading image. This is most likely caused by special characters in "
               "the filename: '{}'".format(filename))
        logger.error(msg)
        if raise_error:
            raise Exception(msg)
    except Exception as err:  # pylint:disable=broad-except
        success = False
        msg = "Failed to load image '{}'. Original Error: {}".format(filename, str(err))
        logger.error(msg)
        if raise_error:
            raise Exception(msg)
    logger.trace("Loaded image: '%s'. Success: %s", filename, success)
    return image


def hash_image_file(filename):
    """ Return an image file's sha1 hash """
    logger = logging.getLogger(__name__)  # pylint:disable=invalid-name
    img = cv2_read_img(filename, raise_error=True)
    img_hash = sha1(img).hexdigest()
    logger.trace("filename: '%s', hash: %s", filename, img_hash)
    return img_hash


def hash_encode_image(image, extension):
    """ Encode the image, get the hash and return the hash with
        encoded image """
    img = cv2.imencode(extension, image)[1]  # pylint:disable=no-member,c-extension-no-member
    f_hash = sha1(
        cv2.imdecode(  # pylint:disable=no-member,c-extension-no-member
            img,
            cv2.IMREAD_UNCHANGED)).hexdigest()  # pylint:disable=no-member,c-extension-no-member
    return f_hash, img


 </s> add  </s> remove             batch = list()
            for _ in range(batchsize):
                img_path = next(img_iter)
                data = self.process_face(img_path, side, is_display)
                batch.append(data)
            batch = list(zip(*batch))
            batch = [np.array(x, dtype="float32") for x in batch]
            logger.trace("Yielding batch: (size: %s, item shapes: %s, side:  '%s', "
                         "is_display: %s)",
                         len(batch), [item.shape for item in batch], side, is_display)
            yield batch

        logger.debug("Finished minibatch generator: (side: '%s', is_display: %s)",
                     side, is_display)

    def process_face(self, filename, side, is_display):
        """ Load an image and perform transformation and warping """
        logger.trace("Process face: (filename: '%s', side: '%s', is_display: %s)",
                     filename, side, is_display)
        image = cv2_read_img(filename, raise_error=True)
        if self.mask_class or self.training_opts["warp_to_landmarks"]:
            src_pts = self.get_landmarks(filename, image, side)
        if self.mask_class:
            image = self.mask_class(src_pts, image, channels=4).mask

        image = self.processing.color_adjust(image,
                                             self.training_opts["augment_color"],
                                             is_display)
        if not is_display:
            image = self.processing.random_transform(image)
            if not self.training_opts["no_flip"]:
                image = self.processing.do_random_flip(image)
        sample = image.copy()[:, :, :3]

        if self.training_opts["warp_to_landmarks"]:
            dst_pts = self.get_closest_match(filename, side, src_pts)
            processed = self.processing.random_warp_landmarks(image, src_pts, dst_pts)
 </s> add             img_paths = [next(img_iter) for _ in range(batchsize)]
            yield self._process_batch(img_paths, side)

        logger.debug("Finished minibatch generator: (side: '%s')", side)

    def _process_batch(self, filenames, side):
        """ Performs the augmentation and compiles target images and samples. See
        :func:`minibatch_ab` for more details on the output. """
        logger.trace("Process batch: (filenames: '%s', side: '%s')", filenames, side)
        batch = read_image_batch(filenames)
        processed = dict()
        to_landmarks = self._training_opts["warp_to_landmarks"]

        # Initialize processing training size on first image
        if not self._processing.initialized:
            self._processing.initialize(batch.shape[1])

        # Get Landmarks prior to manipulating the image
        if self._mask_class or to_landmarks:
            batch_src_pts = self._get_landmarks(filenames, batch, side)

        # Color augmentation before mask is added
        if self._training_opts["augment_color"]:
            batch = self._processing.color_adjust(batch)

        # Add mask to batch prior to transforms and warps
        if self._mask_class:
            batch = np.array([self._mask_class(src_pts, image, channels=4).mask
                              for src_pts, image in zip(batch_src_pts, batch)])

        # Random Transform and flip
        batch = self._processing.transform(batch)
        if not self._training_opts["no_flip"]:
            batch = self._processing.random_flip(batch)

        # Add samples to output if this is for display
        if self._processing.is_display:
            processed["samples"] = batch[..., :3].astype("float32") / 255.0

        # Get Targets
        processed.update(self._processing.get_targets(batch))

        # Random Warp
        if to_landmarks:
            warp_kwargs = dict(batch_src_points=batch_src_pts,
                               batch_dst_points=self._get_closest_match(filenames,
                                                                        side,
                                                                        batch_src_pts)) </s> remove         image = cv2_read_img(filename, raise_error=True)
 </s> add         image = read_image(filename, raise_error=True) </s> remove             retval = cv2_read_img(filename, raise_error=True)
 </s> add             retval = read_image(filename, raise_error=True) </s> remove             image = cv2_read_img(src, raise_error=True)
 </s> add             image = read_image(src, raise_error=True)
            image = read_image(filename, raise_error=False)
      def load_disk_frames(self):
          """ Load frames from disk """
          logger.debug("Input is separate Frames. Loading images")
          for filename in self.input_images:
            image = read_image(filename, raise_error=False)
              if image is None:
                  continue
              yield filename, image
  
      def load_video_frames(self):
keep keep keep keep replace keep keep keep keep keep
 Optimize Data Augmentation (#881)

* Move image utils to lib.image
* Add .pylintrc file
* Remove some cv2 pylint ignores
* TrainingData: Load images from disk in batches
* TrainingData: get_landmarks to batch
* TrainingData: transform and flip to batches
* TrainingData: Optimize color augmentation
* TrainingData: Optimize target and random_warp
* TrainingData - Convert _get_closest_match for batching
* TrainingData: Warp To Landmarks optimized
* Save models to threadpoolexecutor
* Move stack_images, Rename ImageManipulation. ImageAugmentation Docstrings
* Masks: Set dtype and threshold for lib.masks based on input face
* Docstrings and Documentation 
scripts/fsmedia.py