  
      def detect_pnet(self, image, height, width):
          # pylint: disable=too-many-locals
          """ first stage - fast proposal network (pnet) to obtain face candidates """
          scales = calculate_scales(height, width, self.minsize, self.factor)
          rectangles = []
          for scale in scales:
              scale_img = cv2.resize(image,  # pylint:disable=no-member
                                     (int(width * scale), int(height * scale)))
              input_ = scale_img.reshape(1, *scale_img.shape)
              output = self.pnet.predict(input_)
              # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
              # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
              cls_prob = output[0][0][:, :, 1]
              roi = output[1][0]
              out_h, out_w = cls_prob.shape
              out_side = max(out_h, out_w)
              cls_prob = np.swapaxes(cls_prob, 0, 1)
              roi = np.swapaxes(roi, 0, 2)
              rectangle = detect_face_12net(cls_prob,
                                            roi,
 
</s> remove             cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
 </s> add             cls_prob = np.swapaxes(cls_prob, 1, 2)
            roi = np.swapaxes(roi, 1, 3)
            for b in range(batch_items):
                # first index 0 = cls score, 1 = one hot repr
                rectangle = detect_face_12net(cls_prob[b, ...],
                                              roi[b, ...],
                                              out_side,
                                              1 / scale,
                                              width,
                                              height,
                                              self.threshold[0])
                rectangles[b].extend(rectangle)
        return [nms(x, 0.7, 'iou') for x in rectangles]

    def detect_rnet(self, images, rectangle_batch, height, width): </s> remove         total_boxes = np.empty((0, 9))
        points = np.empty(0)
        # TODO Implement batch support
        image = batch[0]
        origin_h, origin_w = image.shape[:2]
        rectangles = self.detect_pnet(image, origin_h, origin_w)
        if not rectangles:
            return total_boxes, points
        rectangles = self.detect_rnet(image, rectangles, origin_h, origin_w)
        if not rectangles:
            return total_boxes, points
        rectangles = self.detect_onet(image, rectangles, origin_h, origin_w)
        if rectangles:
            total_boxes = np.array([result[:5] for result in rectangles])
            points = np.array([result[5:] for result in rectangles]).T
        return total_boxes, points

    def detect_pnet(self, image, height, width):
 </s> add         origin_h, origin_w = batch.shape[1:3]
        rectangles = self.detect_pnet(batch, origin_h, origin_w)
        rectangles = self.detect_rnet(batch, rectangles, origin_h, origin_w)
        rectangles = self.detect_onet(batch, rectangles, origin_h, origin_w)
        ret_boxes = list()
        ret_points = list()
        for rects in rectangles:
            if rects:
                total_boxes = np.array([result[:5] for result in rects])
                points = np.array([result[5:] for result in rects]).T
            else:
                total_boxes = np.empty((0, 9))
                points = np.empty(0)
            ret_boxes.append(total_boxes)
            ret_points.append(points)
        return ret_boxes, ret_points

    def detect_pnet(self, images, height, width): </s> remove         crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
 </s> add         ret = []
        # TODO: batching
        for b, rectangles in enumerate(rectangle_batch):
            if not rectangles:
                ret.append(list())
                continue
            image = images[b]
            crop_number = 0
            predict_24_batch = []
            for rect in rectangles:
                crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
                scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
                predict_24_batch.append(scale_img)
                crop_number += 1
            predict_24_batch = np.array(predict_24_batch)
            output = self.rnet.predict(predict_24_batch, batch_size=128)
            cls_prob = output[0]
            cls_prob = np.array(cls_prob)
            roi_prob = output[1]
            roi_prob = np.array(roi_prob)
            ret.append(filter_face_24net(
                cls_prob, roi_prob, rectangles, width, height, self.threshold[1]
            ))
        return ret

    def detect_onet(self, images, rectangle_batch, height, width): </s> remove         crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])
 </s> add         ret = list()
        # TODO: batching
        for b, rectangles in enumerate(rectangle_batch):
            if not rectangles:
                ret.append(list())
                continue
            image = images[b]
            crop_number = 0
            predict_batch = []
            for rect in rectangles:
                crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
                scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
                predict_batch.append(scale_img)
                crop_number += 1
            predict_batch = np.array(predict_batch)
            output = self.onet.predict(predict_batch, batch_size=128)
            cls_prob = output[0]
            roi_prob = output[1]
            pts_prob = output[2]  # index
            ret.append(filter_face_48net(
                cls_prob,
                roi_prob,
                pts_prob,
                rectangles,
                width,
                height,
                self.threshold[2]
            ))
        return ret </s> remove     def predict(self, feed):
 </s> add     def _amd_predict_with_optimized_batchsizes(self, feed, batch_size):
        """ Minimizes the amount of kernels to be compiled when using
        the ``Amd`` backend with varying batchsizes while trying to keep
        the batchsize as high as possible.

        Parameters
        ----------
        feed: numpy.ndarray or list
            The feed to be provided to the model as input. This should be a ``numpy.ndarray``
            for single inputs or a ``list`` of ``numpy.ndarrays`` for multiple inputs.
        batch_size: int
            The upper batchsize to use.
        """
        if isinstance(feed, np.ndarray):
            feed = [feed]
        items = feed[0].shape[0]
        done_items = 0
        results = list()
        while done_items < items:
            if batch_size < 4:  # Not much difference in BS < 4
                batch_size = 1
            batch_items = ((items - done_items) // batch_size) * batch_size
            if batch_items:
                pred_data = [x[done_items:done_items + batch_items] for x in feed]
                pred = self._model.predict(pred_data, batch_size=batch_size)
                done_items += batch_items
                results.append(pred)
            batch_size //= 2
        if isinstance(results[0], np.ndarray):
            return np.concatenate(results)
        return [np.concatenate(x) for x in zip(*results)]

    def predict(self, feed, batch_size=None):
        if self._pnet_scales is None:
            self._pnet_scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = [[] for _ in range(images.shape[0])]
        batch_items = images.shape[0]
        for scale in self._pnet_scales:
            rwidth, rheight = int(width * scale), int(height * scale)
            batch = np.empty((batch_items, rheight, rwidth, 3), dtype="float32")
            for b in range(batch_items):
                batch[b, ...] = cv2.resize(images[b, ...], (rwidth, rheight))
            output = self.pnet.predict(batch)
            cls_prob = output[0][..., 1]
            roi = output[1]
            out_h, out_w = cls_prob.shape[1:3]
  
      def detect_pnet(self, image, height, width):
          # pylint: disable=too-many-locals
          """ first stage - fast proposal network (pnet) to obtain face candidates """
        if self._pnet_scales is None:
            self._pnet_scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = [[] for _ in range(images.shape[0])]
        batch_items = images.shape[0]
        for scale in self._pnet_scales:
            rwidth, rheight = int(width * scale), int(height * scale)
            batch = np.empty((batch_items, rheight, rwidth, 3), dtype="float32")
            for b in range(batch_items):
                batch[b, ...] = cv2.resize(images[b, ...], (rwidth, rheight))
            output = self.pnet.predict(batch)
            cls_prob = output[0][..., 1]
            roi = output[1]
            out_h, out_w = cls_prob.shape[1:3]
        if self._pnet_scales is None:
            self._pnet_scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = [[] for _ in range(images.shape[0])]
        batch_items = images.shape[0]
        for scale in self._pnet_scales:
            rwidth, rheight = int(width * scale), int(height * scale)
            batch = np.empty((batch_items, rheight, rwidth, 3), dtype="float32")
            for b in range(batch_items):
                batch[b, ...] = cv2.resize(images[b, ...], (rwidth, rheight))
            output = self.pnet.predict(batch)
            cls_prob = output[0][..., 1]
            roi = output[1]
            out_h, out_w = cls_prob.shape[1:3]
        if self._pnet_scales is None:
            self._pnet_scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = [[] for _ in range(images.shape[0])]
        batch_items = images.shape[0]
        for scale in self._pnet_scales:
            rwidth, rheight = int(width * scale), int(height * scale)
            batch = np.empty((batch_items, rheight, rwidth, 3), dtype="float32")
            for b in range(batch_items):
                batch[b, ...] = cv2.resize(images[b, ...], (rwidth, rheight))
            output = self.pnet.predict(batch)
            cls_prob = output[0][..., 1]
            roi = output[1]
            out_h, out_w = cls_prob.shape[1:3]
        if self._pnet_scales is None:
            self._pnet_scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = [[] for _ in range(images.shape[0])]
        batch_items = images.shape[0]
        for scale in self._pnet_scales:
            rwidth, rheight = int(width * scale), int(height * scale)
            batch = np.empty((batch_items, rheight, rwidth, 3), dtype="float32")
            for b in range(batch_items):
                batch[b, ...] = cv2.resize(images[b, ...], (rwidth, rheight))
            output = self.pnet.predict(batch)
            cls_prob = output[0][..., 1]
            roi = output[1]
            out_h, out_w = cls_prob.shape[1:3]
        if self._pnet_scales is None:
            self._pnet_scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = [[] for _ in range(images.shape[0])]
        batch_items = images.shape[0]
        for scale in self._pnet_scales:
            rwidth, rheight = int(width * scale), int(height * scale)
            batch = np.empty((batch_items, rheight, rwidth, 3), dtype="float32")
            for b in range(batch_items):
                batch[b, ...] = cv2.resize(images[b, ...], (rwidth, rheight))
            output = self.pnet.predict(batch)
            cls_prob = output[0][..., 1]
            roi = output[1]
            out_h, out_w = cls_prob.shape[1:3]
        if self._pnet_scales is None:
            self._pnet_scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = [[] for _ in range(images.shape[0])]
        batch_items = images.shape[0]
        for scale in self._pnet_scales:
            rwidth, rheight = int(width * scale), int(height * scale)
            batch = np.empty((batch_items, rheight, rwidth, 3), dtype="float32")
            for b in range(batch_items):
                batch[b, ...] = cv2.resize(images[b, ...], (rwidth, rheight))
            output = self.pnet.predict(batch)
            cls_prob = output[0][..., 1]
            roi = output[1]
            out_h, out_w = cls_prob.shape[1:3]
        if self._pnet_scales is None:
            self._pnet_scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = [[] for _ in range(images.shape[0])]
        batch_items = images.shape[0]
        for scale in self._pnet_scales:
            rwidth, rheight = int(width * scale), int(height * scale)
            batch = np.empty((batch_items, rheight, rwidth, 3), dtype="float32")
            for b in range(batch_items):
                batch[b, ...] = cv2.resize(images[b, ...], (rwidth, rheight))
            output = self.pnet.predict(batch)
            cls_prob = output[0][..., 1]
            roi = output[1]
            out_h, out_w = cls_prob.shape[1:3]
        if self._pnet_scales is None:
            self._pnet_scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = [[] for _ in range(images.shape[0])]
        batch_items = images.shape[0]
        for scale in self._pnet_scales:
            rwidth, rheight = int(width * scale), int(height * scale)
            batch = np.empty((batch_items, rheight, rwidth, 3), dtype="float32")
            for b in range(batch_items):
                batch[b, ...] = cv2.resize(images[b, ...], (rwidth, rheight))
            output = self.pnet.predict(batch)
            cls_prob = output[0][..., 1]
            roi = output[1]
            out_h, out_w = cls_prob.shape[1:3]
        if self._pnet_scales is None:
            self._pnet_scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = [[] for _ in range(images.shape[0])]
        batch_items = images.shape[0]
        for scale in self._pnet_scales:
            rwidth, rheight = int(width * scale), int(height * scale)
            batch = np.empty((batch_items, rheight, rwidth, 3), dtype="float32")
            for b in range(batch_items):
                batch[b, ...] = cv2.resize(images[b, ...], (rwidth, rheight))
            output = self.pnet.predict(batch)
            cls_prob = output[0][..., 1]
            roi = output[1]
            out_h, out_w = cls_prob.shape[1:3]
        if self._pnet_scales is None:
            self._pnet_scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = [[] for _ in range(images.shape[0])]
        batch_items = images.shape[0]
        for scale in self._pnet_scales:
            rwidth, rheight = int(width * scale), int(height * scale)
            batch = np.empty((batch_items, rheight, rwidth, 3), dtype="float32")
            for b in range(batch_items):
                batch[b, ...] = cv2.resize(images[b, ...], (rwidth, rheight))
            output = self.pnet.predict(batch)
            cls_prob = output[0][..., 1]
            roi = output[1]
            out_h, out_w = cls_prob.shape[1:3]
        if self._pnet_scales is None:
            self._pnet_scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = [[] for _ in range(images.shape[0])]
        batch_items = images.shape[0]
        for scale in self._pnet_scales:
            rwidth, rheight = int(width * scale), int(height * scale)
            batch = np.empty((batch_items, rheight, rwidth, 3), dtype="float32")
            for b in range(batch_items):
                batch[b, ...] = cv2.resize(images[b, ...], (rwidth, rheight))
            output = self.pnet.predict(batch)
            cls_prob = output[0][..., 1]
            roi = output[1]
            out_h, out_w = cls_prob.shape[1:3]
        if self._pnet_scales is None:
            self._pnet_scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = [[] for _ in range(images.shape[0])]
        batch_items = images.shape[0]
        for scale in self._pnet_scales:
            rwidth, rheight = int(width * scale), int(height * scale)
            batch = np.empty((batch_items, rheight, rwidth, 3), dtype="float32")
            for b in range(batch_items):
                batch[b, ...] = cv2.resize(images[b, ...], (rwidth, rheight))
            output = self.pnet.predict(batch)
            cls_prob = output[0][..., 1]
            roi = output[1]
            out_h, out_w = cls_prob.shape[1:3]
              out_side = max(out_h, out_w)
              cls_prob = np.swapaxes(cls_prob, 0, 1)
              roi = np.swapaxes(roi, 0, 2)
              rectangle = detect_face_12net(cls_prob,
                                            roi,
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 Optimized mtcnn a bit + added batching (#874) 
plugins/extract/detect/mtcnn.py