      def minibatch(self, q_name, load_thread):
          """ A generator function that yields epoch, batchsize of warped_img
              and batchsize of target_img from the load queue """
          logger.debug("Launching minibatch generator for queue: '%s'", q_name)
          queue = queue_manager.get_queue(q_name)
          while True:
              if load_thread.has_error:
                  logger.debug("Thread error detected")
                  break
              batch = list()
              for _ in range(self.batchsize):
                  images = queue.get()
                  for idx, image in enumerate(images):
                      if len(batch) < idx + 1:
                          batch.append(list())
                      batch[idx].append(image)
              batch = [np.float32(image) for image in batch]
              logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                           len(batch), [item.shape for item in batch], q_name)
              yield batch
          logger.debug("Finished minibatch generator for queue: '%s'", q_name)
          load_thread.join()
  
      def process_face(self, filename, side, is_timelapse):
          """ Load an image and perform transformation and warping """
 
</s> remove         while True:
            if do_shuffle:
                shuffle(images)
            for img in images:
                logger.trace("Putting to batch queue: (q_name: '%s', side: '%s')", q_name, side)
                queue.put(self.process_face(img, side, is_timelapse))
            epoch += 1
 </s> add         def _img_iter(imgs):
            while True:
                if do_shuffle:
                    shuffle(imgs)
                for img in imgs:
                    yield img
        img_iter = _img_iter(images)
        epoch = 0
        for memory_wrapper in mem_gen:
            memory = memory_wrapper.get()
            logger.debug("Putting to batch queue: (q_name: '%s', side: '%s')", q_name, side)
            for i, img_path in enumerate(img_iter):
                imgs = self.process_face(img_path, side, is_timelapse)
                for j, img in enumerate(imgs):
                    memory[j][i][:] = img
                epoch += 1
                if i == batchsize - 1:
                    break
            memory_wrapper.ready() </s> remove         epoch = 0
        queue = queue_manager.get_queue(q_name)
 </s> add  </s> remove     def load_batches(self, images, q_name, side, is_timelapse, do_shuffle=True):
 </s> add     def load_batches(self, mem_gen, images, q_name, side, is_timelapse, do_shuffle=True, batchsize=0): </s> remove         q_size = batchsize * 8
        # Don't use a multiprocessing queue because sometimes the MP Manager borks on numpy arrays
        queue_manager.add_queue(q_name, maxsize=q_size, multiprocessing_queue=False)
        load_thread = MultiThread(self.load_batches,
                                  images,
                                  q_name,
                                  side,
                                  is_timelapse,
                                  do_shuffle)
 </s> add         load_thread = FixedProducerDispatcher(
            self.load_batches,
            batch_shape,
            args=(images, q_name, side, is_timelapse, do_shuffle, batchsize),
            log_queue=queue_manager._log_queue,
            log_level=logger.getEffectiveLevel(),
        )
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
      def minibatch(self, q_name, load_thread):
          """ A generator function that yields epoch, batchsize of warped_img
              and batchsize of target_img from the load queue """
          logger.debug("Launching minibatch generator for queue: '%s'", q_name)
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
        for batch_wrapper in load_thread:
            with batch_wrapper as batch:
                logger.trace("Yielding batch: (size: %s, item shapes: %s, queue:  '%s'",
                             len(batch), [item.shape for item in batch], q_name)
                yield batch
        load_thread.stop()
          logger.debug("Finished minibatch generator for queue: '%s'", q_name)
          load_thread.join()
  
      def process_face(self, filename, side, is_timelapse):
          """ Load an image and perform transformation and warping """
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 Added multithreading dispatcher for training_data 
lib/training_data.py