                  logger.error("Failed to save image '%s'. Original Error: %s", filename, err)
                  continue
          logger.debug("Save Faces: Complete")
  
      def run_extraction(self):
          """ Run Face Detection """
          save_queue = queue_manager.get_queue("extract_out")
          to_process = self.process_item_count()
          frame_no = 0
          size = self.args.size if hasattr(self.args, "size") else 256
          align_eyes = self.args.align_eyes if hasattr(self.args, "align_eyes") else False
  
          if self.plugins.is_parallel:
              logger.debug("Using parallel processing")
              self.plugins.launch_aligner()
              self.plugins.launch_detector()
          if not self.plugins.is_parallel:
              logger.debug("Using serial processing")
              self.run_detection(to_process)
              self.plugins.launch_aligner()
  
          for faces in tqdm(self.plugins.detect_faces(extract_pass="align"),
                            total=to_process,
                            file=sys.stdout,
                            desc="Extracting faces"):
  
              filename = faces["filename"]
  
              self.align_face(faces, align_eyes, size, filename)
              self.post_process.do_actions(faces)
  
              faces_count = len(faces["detected_faces"])
              if faces_count == 0:
                  logger.verbose("No faces were detected in image: %s",
                                 os.path.basename(filename))
  
              if not self.verify_output and faces_count > 1:
                  self.verify_output = True
  
              self.output_faces(filename, faces, save_queue)
  
              frame_no += 1
              if frame_no == self.save_interval:
                  self.alignments.save()
                  frame_no = 0
  
          save_queue.put("EOF")
  
      def process_item_count(self):
          """ Return the number of items to be processedd """
          processed = sum(os.path.basename(frame) in self.alignments.data.keys()
                          for frame in self.images.input_images)
          logger.debug("Items already processed: %s", processed)
 
</s> remove     def run_detection(self, to_process):
        """ Run detection only """
        self.plugins.launch_detector()
        detected_faces = dict()
        for detected in tqdm(self.plugins.detect_faces(extract_pass="detect"),
                             total=to_process,
                             file=sys.stdout,
                             desc="Detecting faces"):
            exception = detected.get("exception", False)
 </s> add     def run_extraction(self):
        """ Run Face Detection """
        to_process = self.process_item_count()
        size = self.args.size if hasattr(self.args, "size") else 256
        align_eyes = self.args.align_eyes if hasattr(self.args, "align_eyes") else False
        exception = False

        for phase in range(self.extractor.passes): </s> remove             detected_faces[filename] = detected
 </s> add         faces_count = len(faces["detected_faces"])
        if faces_count == 0:
            logger.verbose("No faces were detected in image: %s",
                           os.path.basename(filename)) </s> remove         self.threaded_io("reload", detected_faces)
 </s> add         if not self.verify_output and faces_count > 1:
            self.verify_output = True </s> remove 

class Plugins():
    """ Detector and Aligner Plugins and queues """
    def __init__(self, arguments, converter_args=None):
        logger.debug("Initializing %s: (converter_args: %s)",
                     self.__class__.__name__, converter_args)
        self.args = arguments
        self.converter_args = converter_args  # Arguments from converter for on the fly extract
        if converter_args is not None:
            self.loglevel = converter_args["loglevel"]
        else:
            self.loglevel = self.args.loglevel

        self.detector = self.load_detector()
        self.aligner = self.load_aligner()
        self.is_parallel = self.set_parallel_processing()

        self.process_detect = None
        self.process_align = None
        self.add_queues()
        logger.debug("Initialized %s", self.__class__.__name__)

    def set_parallel_processing(self):
        """ Set whether to run detect and align together or separately """
        detector_vram = self.detector.vram
        aligner_vram = self.aligner.vram
        gpu_stats = GPUStats()
        if (detector_vram == 0
                or aligner_vram == 0
                or gpu_stats.device_count == 0):
            logger.debug("At least one of aligner or detector have no VRAM requirement. "
                         "Enabling parallel processing.")
            return True

        if hasattr(self.args, "multiprocess") and not self.args.multiprocess:
            logger.info("NB: Parallel processing disabled.You may get faster "
                        "extraction speeds by enabling it with the -mp switch")
            return False

        required_vram = detector_vram + aligner_vram + 320  # 320MB buffer
        stats = gpu_stats.get_card_most_free()
        free_vram = int(stats["free"])
        logger.verbose("%s - %sMB free of %sMB",
                       stats["device"],
                       free_vram,
                       int(stats["total"]))
        if free_vram <= required_vram:
            logger.warning("Not enough free VRAM for parallel processing. "
                           "Switching to serial")
            return False
        return True

    def add_queues(self):
        """ Add the required processing queues to Queue Manager """
        for task in ("load", "detect", "align", "save"):
            size = 0
            if task == "load" or (not self.is_parallel and task == "detect"):
                size = 100
            if task == "load":
                q_name = "extract_in"
            elif task == "save":
                q_name = "extract_out"
            else:
                q_name = task
            queue_manager.add_queue(q_name, maxsize=size)

    def load_detector(self):
        """ Set global arguments and load detector plugin """
        if not self.converter_args:
            detector_name = self.args.detector.replace("-", "_").lower()
        else:
            detector_name = self.converter_args["detector"]
        logger.debug("Loading Detector: '%s'", detector_name)
        # Rotation
        rotation = self.args.rotate_images if hasattr(self.args, "rotate_images") else None
        # Min acceptable face size:
        min_size = self.args.min_size if hasattr(self.args, "min_size") else 0

        detector = PluginLoader.get_detector(detector_name)(
            loglevel=self.loglevel,
            rotation=rotation,
            min_size=min_size)

        return detector

    def load_aligner(self):
        """ Set global arguments and load aligner plugin """
        if not self.converter_args:
            aligner_name = self.args.aligner.replace("-", "_").lower()
        else:
            aligner_name = self.converter_args["aligner"]

        logger.debug("Loading Aligner: '%s'", aligner_name)

        aligner = PluginLoader.get_aligner(aligner_name)(
            loglevel=self.loglevel)

        return aligner

    def launch_aligner(self):
        """ Launch the face aligner """
        logger.debug("Launching Aligner")
        out_queue = queue_manager.get_queue("align")
        kwargs = {"in_queue": queue_manager.get_queue("detect"),
                  "out_queue": out_queue}

        self.process_align = SpawnProcess(self.aligner.run, **kwargs)
        event = self.process_align.event
        error = self.process_align.error
        self.process_align.start()

        # Wait for Aligner to take it's VRAM
        # The first ever load of the model for FAN has reportedly taken
        # up to 3-4 minutes, hence high timeout.
        # TODO investigate why this is and fix if possible
        for mins in reversed(range(5)):
            for seconds in range(60):
                event.wait(seconds)
                if event.is_set():
                    break
                if error.is_set():
                    break
            if event.is_set():
                break
            if mins == 0 or error.is_set():
                raise ValueError("Error initializing Aligner")
            logger.info("Waiting for Aligner... Time out in %s minutes", mins)

        logger.debug("Launched Aligner")

    def launch_detector(self):
        """ Launch the face detector """
        logger.debug("Launching Detector")
        out_queue = queue_manager.get_queue("detect")
        kwargs = {"in_queue": queue_manager.get_queue("extract_in"),
                  "out_queue": out_queue}
        if self.converter_args:
            kwargs["processes"] = 1
        mp_func = PoolProcess if self.detector.parent_is_pool else SpawnProcess
        self.process_detect = mp_func(self.detector.run, **kwargs)

        event = self.process_detect.event if hasattr(self.process_detect, "event") else None
        error = self.process_detect.error if hasattr(self.process_detect, "error") else None
        self.process_detect.start()

        if event is None:
            logger.debug("Launched Detector")
            return

        for mins in reversed(range(5)):
            for seconds in range(60):
                event.wait(seconds)
                if event.is_set():
                    break
                if error and error.is_set():
                    break
            if event.is_set():
                break
            if mins == 0 or (error and error.is_set()):
                raise ValueError("Error initializing Detector")
            logger.info("Waiting for Detector... Time out in %s minutes", mins)

        logger.debug("Launched Detector")

    def detect_faces(self, extract_pass="detect"):
        """ Detect faces from in an image """
        logger.debug("Running Detection. Pass: '%s'", extract_pass)
        if self.is_parallel or extract_pass == "align":
            out_queue = queue_manager.get_queue("align")
        if not self.is_parallel and extract_pass == "detect":
            out_queue = queue_manager.get_queue("detect")

        while True:
            try:
                faces = out_queue.get(True, 1)
                if faces == "EOF":
                    break
                if isinstance(faces, dict) and faces.get("exception"):
                    pid = faces["exception"][0]
                    t_back = faces["exception"][1].getvalue()
                    err = "Error in child process {}. {}".format(pid, t_back)
                    raise Exception(err)
            except QueueEmpty:
                continue

            yield faces
        logger.debug("Detection Complete")
 </s> add  </s> remove             del detected["image"]
            filename = detected["filename"]
 </s> add     def output_processing(self, faces, align_eyes, size, filename):
        """ Prepare faces for output """
        self.align_face(faces, align_eyes, size, filename)
        self.post_process.do_actions(faces)
                  logger.error("Failed to save image '%s'. Original Error: %s", filename, err)
                  continue
          logger.debug("Save Faces: Complete")
  












































      def process_item_count(self):
          """ Return the number of items to be processedd """
          processed = sum(os.path.basename(frame) in self.alignments.data.keys()
                          for frame in self.images.input_images)
          logger.debug("Items already processed: %s", processed)
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 Face filter vgg face (#724)

* Implement extraction pipeline

* Face filter to vgg_face. Resume partial model downloads

* On-the-fly conversion to extraction pipeline

* Move git model ids from get_model to model definition 
scripts/extract.py