               group=1,
               biased=True):
          """ Conv Layer """
          # pylint: disable=too-many-locals
  
          # Verify that the padding is acceptable
          self.validate_padding(padding)
          # Get the number of channels in the input
          c_i = int(inp.get_shape()[-1])
          # Verify that the grouping parameter is valid
          assert c_i % group == 0
          assert c_o % group == 0
          # Convolution for a given input and kernel
          convolve = lambda i, k: tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding) # noqa
          with tf.variable_scope(name) as scope:
              kernel = self.make_var('weights',
                                     shape=[k_h, k_w, c_i // group, c_o])
              # This is the common-case. Convolve the input without any
              # further complications.
              output = convolve(inp, kernel)
              # Add the biases
              if biased:
                  biases = self.make_var('biases', [c_o])
                  output = tf.nn.bias_add(output, biases)
              if relu:
                  # ReLU non-linearity
                  output = tf.nn.relu(output, name=scope.name)
              return output
  
      @layer
      def prelu(self, inp, name):
          """ Prelu Layer """
          with tf.variable_scope(name):
              i = int(inp.get_shape()[-1])
              alpha = self.make_var('alpha', shape=(i,))
              output = tf.nn.relu(inp) + tf.multiply(alpha, -tf.nn.relu(-inp))
          return output
  
      @layer
      def max_pool(self, inp, k_h, k_w,  # pylint: disable=too-many-arguments
                   s_h, s_w, name, padding='SAME'):
          """ Max Pool Layer """
          self.validate_padding(padding)
          return tf.nn.max_pool(inp,
                                ksize=[1, k_h, k_w, 1],
                                strides=[1, s_h, s_w, 1],
                                padding=padding,
                                name=name)
  
      @layer
      def fc(self, inp, num_out, name, relu=True):  # pylint: disable=invalid-name
          """ FC Layer """
          with tf.variable_scope(name):
              input_shape = inp.get_shape()
              if input_shape.ndims == 4:
                  # The input is spatial. Vectorize it first.
                  dim = 1
                  for this_dim in input_shape[1:].as_list():
                      dim *= int(this_dim)
                  feed_in = tf.reshape(inp, [-1, dim])
              else:
                  feed_in, dim = (inp, input_shape[-1].value)
              weights = self.make_var('weights', shape=[dim, num_out])
              biases = self.make_var('biases', [num_out])
              operator = tf.nn.relu_layer if relu else tf.nn.xw_plus_b
              fc = operator(feed_in, weights, biases, name=name)  # pylint: disable=invalid-name
              return fc
  
      @layer
      def softmax(self, target, axis, name=None):  # pylint: disable=no-self-use
          """ Multi dimensional softmax,
              refer to https://github.com/tensorflow/tensorflow/issues/210
              compute softmax along the dimension of target
              the native softmax only supports batch_size x dimension """
  
          max_axis = tf.reduce_max(target, axis, keepdims=True)
          target_exp = tf.exp(target-max_axis)
          normalize = tf.reduce_sum(target_exp, axis, keepdims=True)
          softmax = tf.div(target_exp, normalize, name)
          return softmax
  
  
  class PNet(Network):
      """ Tensorflow PNet """
      def setup(self):
          (self.feed('data')  # pylint: disable=no-value-for-parameter, no-member
           .conv(3, 3, 10, 1, 1, padding='VALID', relu=False, name='conv1')
           .prelu(name='PReLU1')
           .max_pool(2, 2, 2, 2, name='pool1')
           .conv(3, 3, 16, 1, 1, padding='VALID', relu=False, name='conv2')
           .prelu(name='PReLU2')
           .conv(3, 3, 32, 1, 1, padding='VALID', relu=False, name='conv3')
           .prelu(name='PReLU3')
           .conv(1, 1, 2, 1, 1, relu=False, name='conv4-1')
           .softmax(3, name='prob1'))
  
          (self.feed('PReLU3')  # pylint: disable=no-value-for-parameter
           .conv(1, 1, 4, 1, 1, relu=False, name='conv4-2'))
  
  
  class RNet(Network):
      """ Tensorflow RNet """
      def setup(self):
          (self.feed('data')  # pylint: disable=no-value-for-parameter, no-member
           .conv(3, 3, 28, 1, 1, padding='VALID', relu=False, name='conv1')
           .prelu(name='prelu1')
           .max_pool(3, 3, 2, 2, name='pool1')
           .conv(3, 3, 48, 1, 1, padding='VALID', relu=False, name='conv2')
           .prelu(name='prelu2')
           .max_pool(3, 3, 2, 2, padding='VALID', name='pool2')
           .conv(2, 2, 64, 1, 1, padding='VALID', relu=False, name='conv3')
           .prelu(name='prelu3')
           .fc(128, relu=False, name='conv4')
           .prelu(name='prelu4')
           .fc(2, relu=False, name='conv5-1')
           .softmax(1, name='prob1'))
  
          (self.feed('prelu4')  # pylint: disable=no-value-for-parameter
           .fc(4, relu=False, name='conv5-2'))
  
  
  class ONet(Network):
      """ Tensorflow ONet """
      def setup(self):
          (self.feed('data')  # pylint: disable=no-value-for-parameter, no-member
           .conv(3, 3, 32, 1, 1, padding='VALID', relu=False, name='conv1')
           .prelu(name='prelu1')
           .max_pool(3, 3, 2, 2, name='pool1')
           .conv(3, 3, 64, 1, 1, padding='VALID', relu=False, name='conv2')
           .prelu(name='prelu2')
           .max_pool(3, 3, 2, 2, padding='VALID', name='pool2')
           .conv(3, 3, 64, 1, 1, padding='VALID', relu=False, name='conv3')
           .prelu(name='prelu3')
           .max_pool(2, 2, 2, 2, name='pool3')
           .conv(2, 2, 128, 1, 1, padding='VALID', relu=False, name='conv4')
           .prelu(name='prelu4')
           .fc(256, relu=False, name='conv5')
           .prelu(name='prelu5')
           .fc(2, relu=False, name='conv6-1')
           .softmax(1, name='prob1'))
  
          (self.feed('prelu5')  # pylint: disable=no-value-for-parameter
           .fc(4, relu=False, name='conv6-2'))
  
          (self.feed('prelu5')  # pylint: disable=no-value-for-parameter
           .fc(10, relu=False, name='conv6-3'))
  
  
  def create_mtcnn(sess, model_path):
      """ Create the network """
      if not model_path:
          model_path, _ = os.path.split(os.path.realpath(__file__))
  
      with tf.variable_scope('pnet'):
          data = tf.placeholder(tf.float32, (None, None, None, 3), 'input')
          pnet = PNet({'data': data})
          pnet.load(model_path[0], sess)
      with tf.variable_scope('rnet'):
          data = tf.placeholder(tf.float32, (None, 24, 24, 3), 'input')
          rnet = RNet({'data': data})
          rnet.load(model_path[1], sess)
      with tf.variable_scope('onet'):
          data = tf.placeholder(tf.float32, (None, 48, 48, 3), 'input')
          onet = ONet({'data': data})
          onet.load(model_path[2], sess)
  
      pnet_fun = lambda img: sess.run(('pnet/conv4-2/BiasAdd:0', # noqa
                                       'pnet/prob1:0'),
                                      feed_dict={'pnet/input:0': img})
      rnet_fun = lambda img: sess.run(('rnet/conv5-2/conv5-2:0', # noqa
                                       'rnet/prob1:0'),
                                      feed_dict={'rnet/input:0': img})
      onet_fun = lambda img: sess.run(('onet/conv6-2/conv6-2:0', # noqa
                                       'onet/conv6-3/conv6-3:0',
                                       'onet/prob1:0'),
                                      feed_dict={'onet/input:0': img})
      return pnet_fun, rnet_fun, onet_fun
  
  
  def detect_face(img, minsize, pnet, rnet,  # pylint: disable=too-many-arguments
                  onet, threshold, factor):
      """Detects faces in an image, and returns bounding boxes and points for them.
      img: input image
      minsize: minimum faces' size
      pnet, rnet, onet: caffemodel
      threshold: threshold=[th1, th2, th3], th1-3 are three steps's threshold
      factor: the factor used to create a scaling pyramid of face sizes to
              detect in the image.
      """
      # pylint: disable=too-many-locals,too-many-statements,too-many-branches
      factor_count = 0
      total_boxes = np.empty((0, 9))
      points = np.empty(0)
 
</s> remove         ident = sum(t.startswith(prefix) for t, _ in self.layers.items()) + 1
        return '%s_%d' % (prefix, ident)

    def make_var(self, name, shape):
        """Creates a new TensorFlow variable."""
        return tf.get_variable(name, shape, trainable=self.trainable)

    @staticmethod
    def validate_padding(padding):
        """Verifies that the padding is one of the supported ones."""
        assert padding in ('SAME', 'VALID')

    @layer
    def conv(self,  # pylint: disable=too-many-arguments
             inp,
             k_h,
             k_w,
             c_o,
             s_h,
             s_w,
             name,
             relu=True,
             padding='SAME',
             group=1,
             biased=True):
        """ Conv Layer """
 </s> add         total_boxes = np.empty((0, 9))
        points = np.empty(0)
        # TODO Implement batch support
        image = batch[0]
        origin_h, origin_w = image.shape[:2]
        rectangles = self.detect_pnet(image, origin_h, origin_w)
        if not rectangles:
            return total_boxes, points
        rectangles = self.detect_rnet(image, rectangles, origin_h, origin_w)
        if not rectangles:
            return total_boxes, points
        rectangles = self.detect_onet(image, rectangles, origin_h, origin_w)
        if rectangles:
            total_boxes = np.array([result[:5] for result in rectangles])
            points = np.array([result[5:] for result in rectangles]).T
        return total_boxes, points

    def detect_pnet(self, image, height, width): </s> remove     def load_graph(self):
        """ Load the tensorflow Model and weights """
        # pylint: disable=not-context-manager
        logger.verbose("Initializing S3FD Network model...")
        with self.tf.gfile.GFile(self.model_path, "rb") as gfile:
            graph_def = self.tf.GraphDef()
            graph_def.ParseFromString(gfile.read())
        fa_graph = self.tf.Graph()
        with fa_graph.as_default():
            self.tf.import_graph_def(graph_def, name="s3fd")
        return fa_graph

    def get_outputs(self):
        """ Return the output tensors """
        tensor_names = ["concat_31", "transpose_72", "transpose_75", "transpose_78",
                        "transpose_81", "transpose_84", "transpose_87", "transpose_90",
                        "transpose_93", "transpose_96", "transpose_99", "transpose_102"]
        logger.debug("tensor_names: %s", tensor_names)
        tensors = [self.graph.get_tensor_by_name("s3fd/{}:0".format(t_name))
                   for t_name in tensor_names]
        logger.debug("tensors: %s", tensors)
        return tensors

    def set_session(self, target_size, vram_ratio, card_id):
        """ Set the TF Session and initialize """
        # pylint: disable=not-context-manager, no-member
        placeholder = np.zeros((1, 3, target_size[0], target_size[1]))
        config = self.tf.ConfigProto()
        if card_id != -1:
            config.gpu_options.visible_device_list = str(card_id)
        if vram_ratio != 1.0:
            config.gpu_options.per_process_gpu_memory_fraction = vram_ratio

        with self.graph.as_default():
            session = self.tf.Session(config=config)
            self.is_gpu = any("gpu" in str(device).lower() for device in session.list_devices())
            session.run(self.output, feed_dict={self.input: placeholder})
        return session

    def detect_face(self, feed_item):
        """ Detect faces """
        feed_item = feed_item - np.array([104.0, 117.0, 123.0])
        feed_item = feed_item.transpose(2, 0, 1)
        feed_item = feed_item.reshape((1,) + feed_item.shape).astype('float32')
        bboxlist = self.session.run(self.output, feed_dict={self.input: feed_item})
        bboxlist = self.post_process(bboxlist)

        keep = self.nms(bboxlist, 0.3)
        bboxlist = bboxlist[keep, :]
        bboxlist = [x for x in bboxlist if x[-1] >= self.confidence]
 </s> add     @staticmethod
    def prepare_batch(batch):
        """ Prepare a batch for prediction """
        batch = batch - np.array([104.0, 117.0, 123.0])
        batch = batch.transpose(0, 3, 1, 2)
        return batch </s> remove         self.model = None

    def initialize(self, *args, **kwargs):
        """ Initialization tasks to run prior to alignments """
        try:
            super().initialize(*args, **kwargs)
            logger.info("Initializing cv2 DNN Aligner...")
            logger.debug("cv2 DNN initialize: (args: %s kwargs: %s)", args, kwargs)
            logger.verbose("Using CPU for alignment")

            self.model = cv2.dnn.readNetFromTensorflow(  # pylint: disable=no-member
                self.model_path)
            self.model.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU)  # pylint: disable=no-member
            self.init.set()
            logger.info("Initialized cv2 DNN Aligner.")
        except Exception as err:
            self.error.set()
            raise err

    def align_image(self, detected_face, image):
 </s> add         self.batchsize = 1

    def init_model(self):
        """ Initialize CV2 DNN Detector Model"""
        self.model = cv2.dnn.readNetFromTensorflow(self.model_path)  # pylint: disable=no-member
        self.model.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU)  # pylint: disable=no-member

    def process_input(self, batch):
        """ Compile the detected faces for prediction """
        faces, batch["roi"] = self.align_image(batch["detected_faces"])
        faces = self._normalize_faces(faces)
        batch["feed"] = np.array(faces, dtype="float32").transpose((0, 3, 1, 2))
        return batch

    def align_image(self, detected_faces): </s> remove         return dict(image=image, center=center, scale=scale)
 </s> add         faces = self._normalize_faces(faces)
        batch["feed"] = np.array(faces, dtype="float32").transpose((0, 3, 1, 2)) / 255.0
        return batch </s> remove     # pylint: disable=too-many-locals,too-many-statements,too-many-branches
 </s> add 
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
               group=1,
               biased=True):
          """ Conv Layer """
          # pylint: disable=too-many-locals
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
        """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale
      """
      # pylint: disable=too-many-locals,too-many-statements,too-many-branches
      factor_count = 0
      total_boxes = np.empty((0, 9))
      points = np.empty(0)
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 De-Multiprocess Extract (#871)

* requirements.txt: - Pin opencv to 4.1.1 (fixes cv2-dnn error)

* lib.face_detect.DetectedFace: change LandmarksXY to landmarks_xy. Add left, right, top, bottom attributes

* lib.model.session: Session manager for loading models into different graphs (for Nvidia + CPU)

* plugins.extract._base: New parent class for all extract plugins

* plugins.extract.pipeline. Remove MultiProcessing. Dynamically limit batchsize for Nvidia cards. Remove loglevel input

* S3FD + FAN plugins. Standardise to Keras version for all backends

* Standardize all extract plugins to new threaded codebase

* Documentation. Start implementing Numpy style docstrings for Sphinx Documentation

* Remove s3fd_amd. Change convert OTF to expect DetectedFace object

* faces_detect - clean up and documentation

* Remove PoolProcess

* Migrate manual tool to new extract workflow

* Remove AMD specific extractor code from cli and plugins

* Sort tool to new extract workflow

* Remove multiprocessing from project

* Remove multiprocessing queues from QueueManager

* Remove multiprocessing support from logger

* Move face_filter to new extraction pipeline

* Alignments landmarksXY > landmarks_xy and legacy handling

* Intercept get_backend for sphinx doc build

# Add Sphinx documentation 
plugins/extract/detect/mtcnn.py