          save_averages = self.get_save_averages()
          backup_func = self.backup.backup_model if self.should_backup(save_averages) else None
          if backup_func:
              logger.info("Backing up models...")
          save_threads = list()
          for network in self.networks.values():
              name = "save_{}".format(network.name)
              save_threads.append(MultiThread(network.save,
                                              name=name,
                                              backup_func=backup_func))
          save_threads.append(MultiThread(self.state.save,
                                          name="save_state",
                                          backup_func=backup_func))
          for thread in save_threads:
              thread.start()
          for thread in save_threads:
              if thread.has_error:
                  logger.error(thread.errors[0])
              thread.join()
          msg = "[Saved models]"
          if save_averages:
              lossmsg = ["{}_{}: {:.5f}".format(self.state.loss_names[side][0],
                                                side.capitalize(),
                                                save_averages[side])
 
</s> remove def rotate_landmarks(face, rotation_matrix):
    # pylint:disable=c-extension-no-member
    """ Rotate the landmarks and bounding box for faces
        found in rotated images.
        Pass in a DetectedFace object or Alignments dict """
    logger = logging.getLogger(__name__)  # pylint:disable=invalid-name
    logger.trace("Rotating landmarks: (rotation_matrix: %s, type(face): %s",
                 rotation_matrix, type(face))
    rotated_landmarks = None
    # Detected Face Object
    if isinstance(face, DetectedFace):
        bounding_box = [[face.x, face.y],
                        [face.x + face.w, face.y],
                        [face.x + face.w, face.y + face.h],
                        [face.x, face.y + face.h]]
        landmarks = face.landmarks_xy

    # Alignments Dict
    elif isinstance(face, dict) and "x" in face:
        bounding_box = [[face.get("x", 0), face.get("y", 0)],
                        [face.get("x", 0) + face.get("w", 0),
                         face.get("y", 0)],
                        [face.get("x", 0) + face.get("w", 0),
                         face.get("y", 0) + face.get("h", 0)],
                        [face.get("x", 0),
                         face.get("y", 0) + face.get("h", 0)]]
        landmarks = face.get("landmarks_xy", list())

    else:
        raise ValueError("Unsupported face type")

    logger.trace("Original landmarks: %s", landmarks)

    rotation_matrix = cv2.invertAffineTransform(  # pylint:disable=no-member
        rotation_matrix)
    rotated = list()
    for item in (bounding_box, landmarks):
        if not item:
            continue
        points = np.array(item, np.int32)
        points = np.expand_dims(points, axis=0)
        transformed = cv2.transform(points,  # pylint:disable=no-member
                                    rotation_matrix).astype(np.int32)
        rotated.append(transformed.squeeze())

    # Bounding box should follow x, y planes, so get min/max
    # for non-90 degree rotations
    pt_x = min([pnt[0] for pnt in rotated[0]])
    pt_y = min([pnt[1] for pnt in rotated[0]])
    pt_x1 = max([pnt[0] for pnt in rotated[0]])
    pt_y1 = max([pnt[1] for pnt in rotated[0]])
    width = pt_x1 - pt_x
    height = pt_y1 - pt_y

    if isinstance(face, DetectedFace):
        face.x = int(pt_x)
        face.y = int(pt_y)
        face.w = int(width)
        face.h = int(height)
        face.r = 0
        if len(rotated) > 1:
            rotated_landmarks = [tuple(point) for point in rotated[1].tolist()]
            face.landmarks_xy = rotated_landmarks
    else:
        face["left"] = int(pt_x)
        face["top"] = int(pt_y)
        face["right"] = int(pt_x1)
        face["bottom"] = int(pt_y1)
        rotated_landmarks = face

    logger.trace("Rotated landmarks: %s", rotated_landmarks)
    return face


 </s> add  </s> remove         return src_points

    def get_closest_match(self, filename, side, src_points):
        """ Return closest matched landmarks from opposite set """
        logger.trace("Retrieving closest matched landmarks: (filename: '%s', src_points: '%s'",
                     filename, src_points)
        landmarks = self.landmarks["a"] if side == "b" else self.landmarks["b"]
        closest_hashes = self._nearest_landmarks.get(filename)
        if not closest_hashes:
            dst_points_items = list(landmarks.items())
            dst_points = list(x[1] for x in dst_points_items)
 </s> add         return np.array(src_points)

    def _get_closest_match(self, filenames, side, batch_src_points):
        """ Only called if the config item ``warp_to_landmarks`` is ``True``. Gets the closest
        matched 68 point landmarks from the opposite training set. """
        logger.trace("Retrieving closest matched landmarks: (filenames: '%s', src_points: '%s'",
                     filenames, batch_src_points)
        landmarks = self._landmarks["a"] if side == "b" else self._landmarks["b"]
        closest_hashes = [self._nearest_landmarks.get(filename) for filename in filenames]
        if None in closest_hashes:
            closest_hashes = self._cache_closest_hashes(filenames, batch_src_points, landmarks)

        batch_dst_points = np.array([landmarks[choice(hsh)] for hsh in closest_hashes])
        logger.trace("Returning: (batch_dst_points: %s)", batch_dst_points.shape)
        return batch_dst_points

    def _cache_closest_hashes(self, filenames, batch_src_points, landmarks):
        """ Cache the nearest landmarks for this batch """
        logger.trace("Caching closest hashes")
        dst_landmarks = list(landmarks.items())
        dst_points = np.array([lm[1] for lm in dst_landmarks])
        batch_closest_hashes = list()

        for filename, src_points in zip(filenames, batch_src_points): </s> remove def full_path_split(path):
    """ Split a given path into all of it's separate components """
    logger = logging.getLogger(__name__)  # pylint:disable=invalid-name
    allparts = list()
    while True:
        parts = os.path.split(path)
        if parts[0] == path:   # sentinel for absolute paths
            allparts.insert(0, parts[0])
            break
        elif parts[1] == path:  # sentinel for relative paths
            allparts.insert(0, parts[1])
            break
        else:
            path = parts[0]
            allparts.insert(0, parts[1])
    logger.trace("path: %s, allparts: %s", path, allparts)
    return allparts


def cv2_read_img(filename, raise_error=False):
    """ Read an image with cv2 and check that an image was actually loaded.
        Logs an error if the image returned is None. or an error has occured.

        Pass raise_error=True if error should be raised """
    logger = logging.getLogger(__name__)  # pylint:disable=invalid-name
    logger.trace("Requested image: '%s'", filename)
    success = True
    image = None
    try:
        image = cv2.imread(filename)  # pylint:disable=no-member,c-extension-no-member
        if image is None:
            raise ValueError
    except TypeError:
        success = False
        msg = "Error while reading image (TypeError): '{}'".format(filename)
        logger.error(msg)
        if raise_error:
            raise Exception(msg)
    except ValueError:
        success = False
        msg = ("Error while reading image. This is most likely caused by special characters in "
               "the filename: '{}'".format(filename))
        logger.error(msg)
        if raise_error:
            raise Exception(msg)
    except Exception as err:  # pylint:disable=broad-except
        success = False
        msg = "Failed to load image '{}'. Original Error: {}".format(filename, str(err))
        logger.error(msg)
        if raise_error:
            raise Exception(msg)
    logger.trace("Loaded image: '%s'. Success: %s", filename, success)
    return image


def hash_image_file(filename):
    """ Return an image file's sha1 hash """
    logger = logging.getLogger(__name__)  # pylint:disable=invalid-name
    img = cv2_read_img(filename, raise_error=True)
    img_hash = sha1(img).hexdigest()
    logger.trace("filename: '%s', hash: %s", filename, img_hash)
    return img_hash


def hash_encode_image(image, extension):
    """ Encode the image, get the hash and return the hash with
        encoded image """
    img = cv2.imencode(extension, image)[1]  # pylint:disable=no-member,c-extension-no-member
    f_hash = sha1(
        cv2.imdecode(  # pylint:disable=no-member,c-extension-no-member
            img,
            cv2.IMREAD_UNCHANGED)).hexdigest()  # pylint:disable=no-member,c-extension-no-member
    return f_hash, img


 </s> add  </s> remove             predictions = [preds["{}_{}".format(side, side)],
 </s> add             predictions = [preds["{0}_{0}".format(side)], </s> remove             headers[side] = self.get_headers(side, other_side, display[0].shape[1])
 </s> add             headers[side] = self.get_headers(side, display[0].shape[1])
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
          save_averages = self.get_save_averages()
          backup_func = self.backup.backup_model if self.should_backup(save_averages) else None
          if backup_func:
              logger.info("Backing up models...")
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
        executor = futures.ThreadPoolExecutor()
        save_threads = [executor.submit(network.save, backup_func=backup_func)
                        for network in self.networks.values()]
        save_threads.append(executor.submit(self.state.save, backup_func=backup_func))
        futures.wait(save_threads)
        # call result() to capture errors
        _ = [thread.result() for thread in save_threads]
          msg = "[Saved models]"
          if save_averages:
              lossmsg = ["{}_{}: {:.5f}".format(self.state.loss_names[side][0],
                                                side.capitalize(),
                                                save_averages[side])
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 Optimize Data Augmentation (#881)

* Move image utils to lib.image
* Add .pylintrc file
* Remove some cv2 pylint ignores
* TrainingData: Load images from disk in batches
* TrainingData: get_landmarks to batch
* TrainingData: transform and flip to batches
* TrainingData: Optimize color augmentation
* TrainingData: Optimize target and random_warp
* TrainingData - Convert _get_closest_match for batching
* TrainingData: Warp To Landmarks optimized
* Save models to threadpoolexecutor
* Move stack_images, Rename ImageManipulation. ImageAugmentation Docstrings
* Masks: Set dtype and threshold for lib.masks based on input face
* Docstrings and Documentation 
plugins/train/model/_base.py