      while minl >= 12:
          scales += [var_m * np.power(factor, factor_count)]
          minl = minl * factor
          factor_count += 1
  
  
  def rect2square(rectangles):
      """ change rectangles into squares (matrix version)
      Input:
 
</s> remove     total_boxes = np.empty((0, 9))
    points = np.empty(0)
    height = img.shape[0]
    width = img.shape[1]
 </s> add  </s> remove     # # # # # # # # # # # # #
    # first stage - fast proposal network (pnet) to obtain face candidates
    # # # # # # # # # # # # #
    for scale in scales:
        height_scale = int(np.ceil(height * scale))
        width_scale = int(np.ceil(width * scale))
        im_data = imresample(img, (height_scale, width_scale))
        im_data = (im_data - 127.5) * 0.0078125
        img_x = np.expand_dims(im_data, 0)
        img_y = np.transpose(img_x, (0, 2, 1, 3))
        out = pnet(img_y)
        out0 = np.transpose(out[0], (0, 2, 1, 3))
        out1 = np.transpose(out[1], (0, 2, 1, 3))

        boxes, _ = generate_bounding_box(out1[0, :, :, 1].copy(),
                                         out0[0, :, :, :].copy(),
                                         scale, threshold[0])

        # inter-scale nms
        pick = nms(boxes.copy(), 0.5, 'Union')
        if boxes.size > 0 and pick.size > 0:
            boxes = boxes[pick, :]
            total_boxes = np.append(total_boxes, boxes, axis=0)

    numbox = total_boxes.shape[0]
    if numbox > 0:
        pick = nms(total_boxes.copy(), 0.7, 'Union')
        total_boxes = total_boxes[pick, :]
        regw = total_boxes[:, 2]-total_boxes[:, 0]
        regh = total_boxes[:, 3]-total_boxes[:, 1]
        qq_1 = total_boxes[:, 0]+total_boxes[:, 5] * regw
        qq_2 = total_boxes[:, 1]+total_boxes[:, 6] * regh
        qq_3 = total_boxes[:, 2]+total_boxes[:, 7] * regw
        qq_4 = total_boxes[:, 3]+total_boxes[:, 8] * regh
        total_boxes = np.transpose(np.vstack([qq_1, qq_2, qq_3, qq_4, total_boxes[:, 4]]))
        total_boxes = rerec(total_boxes.copy())
        total_boxes[:, 0:4] = np.fix(total_boxes[:, 0:4]).astype(np.int32)
        d_y, ed_y, d_x, ed_x, var_y, e_y, var_x, e_x, tmpw, tmph = pad(total_boxes.copy(),
                                                                       width, height)

    numbox = total_boxes.shape[0]

    # # # # # # # # # # # # #
    # second stage - refinement of face candidates with rnet
    # # # # # # # # # # # # #

    if numbox > 0:
        tempimg = np.zeros((24, 24, 3, numbox))
        for k in range(0, numbox):
            tmp = np.zeros((int(tmph[k]), int(tmpw[k]), 3))
            tmp[d_y[k] - 1:ed_y[k], d_x[k] - 1:ed_x[k], :] = img[var_y[k] - 1:e_y[k],
                                                                 var_x[k]-1:e_x[k], :]
            if tmp.shape[0] > 0 and tmp.shape[1] > 0 or tmp.shape[0] == 0 and tmp.shape[1] == 0:
                tempimg[:, :, :, k] = imresample(tmp, (24, 24))
            else:
                return np.empty()
        tempimg = (tempimg-127.5)*0.0078125
        tempimg1 = np.transpose(tempimg, (3, 1, 0, 2))
        out = rnet(tempimg1)
        out0 = np.transpose(out[0])
        out1 = np.transpose(out[1])
        score = out1[1, :]
        ipass = np.where(score > threshold[1])
        total_boxes = np.hstack([total_boxes[ipass[0], 0:4].copy(),
                                 np.expand_dims(score[ipass].copy(), 1)])
        m_v = out0[:, ipass[0]]
        if total_boxes.shape[0] > 0:
            pick = nms(total_boxes, 0.7, 'Union')
            total_boxes = total_boxes[pick, :]
            total_boxes = bbreg(total_boxes.copy(), np.transpose(m_v[:, pick]))
            total_boxes = rerec(total_boxes.copy())

    numbox = total_boxes.shape[0]

    # # # # # # # # # # # # #
    # third stage - further refinement and facial landmarks positions with onet
    # NB: Facial landmarks code commented out for faceswap
    # # # # # # # # # # # # #

    if numbox > 0:
        # third stage
        total_boxes = np.fix(total_boxes).astype(np.int32)
        d_y, ed_y, d_x, ed_x, var_y, e_y, var_x, e_x, tmpw, tmph = pad(total_boxes.copy(),
                                                                       width, height)
        tempimg = np.zeros((48, 48, 3, numbox))
        for k in range(0, numbox):
            tmp = np.zeros((int(tmph[k]), int(tmpw[k]), 3))
            tmp[d_y[k] - 1:ed_y[k], d_x[k] - 1:ed_x[k], :] = img[var_y[k] - 1:e_y[k],
                                                                 var_x[k] - 1:e_x[k], :]
            if tmp.shape[0] > 0 and tmp.shape[1] > 0 or tmp.shape[0] == 0 and tmp.shape[1] == 0:
                tempimg[:, :, :, k] = imresample(tmp, (48, 48))
            else:
                return np.empty()
        tempimg = (tempimg-127.5)*0.0078125
        tempimg1 = np.transpose(tempimg, (3, 1, 0, 2))
        out = onet(tempimg1)
        out0 = np.transpose(out[0])
        out1 = np.transpose(out[1])
        out2 = np.transpose(out[2])
        score = out2[1, :]
        points = out1
        ipass = np.where(score > threshold[2])
        points = points[:, ipass[0]]
        total_boxes = np.hstack([total_boxes[ipass[0], 0:4].copy(),
                                 np.expand_dims(score[ipass].copy(), 1)])
        m_v = out0[:, ipass[0]]

        width = total_boxes[:, 2] - total_boxes[:, 0] + 1
        height = total_boxes[:, 3] - total_boxes[:, 1] + 1
        points[0:5, :] = (np.tile(width, (5, 1)) * points[0:5, :] +
                          np.tile(total_boxes[:, 0], (5, 1)) - 1)
        points[5:10, :] = (np.tile(height, (5, 1)) * points[5:10, :] +
                           np.tile(total_boxes[:, 1], (5, 1)) - 1)
        if total_boxes.shape[0] > 0:
            total_boxes = bbreg(total_boxes.copy(), np.transpose(m_v))
            pick = nms(total_boxes.copy(), 0.7, 'Min')
            total_boxes = total_boxes[pick, :]
            points = points[:, pick]

    return total_boxes, points


# function [boundingbox] = bbreg(boundingbox,reg)
def bbreg(boundingbox, reg):
    """Calibrate bounding boxes"""
    if reg.shape[1] == 1:
        reg = np.reshape(reg, (reg.shape[2], reg.shape[3]))

    width = boundingbox[:, 2] - boundingbox[:, 0] + 1
    height = boundingbox[:, 3] - boundingbox[:, 1] + 1
    b_1 = boundingbox[:, 0] + reg[:, 0] * width
    b_2 = boundingbox[:, 1] + reg[:, 1] * height
    b_3 = boundingbox[:, 2] + reg[:, 2] * width
    b_4 = boundingbox[:, 3] + reg[:, 3] * height
    boundingbox[:, 0:4] = np.transpose(np.vstack([b_1, b_2, b_3, b_4]))
    return boundingbox


def generate_bounding_box(imap, reg, scale, threshold):
    """Use heatmap to generate bounding boxes"""
    # pylint: disable=too-many-locals
    stride = 2
    cellsize = 12

    imap = np.transpose(imap)
    d_x1 = np.transpose(reg[:, :, 0])
    d_y1 = np.transpose(reg[:, :, 1])
    d_x2 = np.transpose(reg[:, :, 2])
    d_y2 = np.transpose(reg[:, :, 3])
    dim_y, dim_x = np.where(imap >= threshold)
    if dim_y.shape[0] == 1:
        d_x1 = np.flipud(d_x1)
        d_y1 = np.flipud(d_y1)
        d_x2 = np.flipud(d_x2)
        d_y2 = np.flipud(d_y2)
    score = imap[(dim_y, dim_x)]
    reg = np.transpose(np.vstack([d_x1[(dim_y, dim_x)], d_y1[(dim_y, dim_x)],
                                  d_x2[(dim_y, dim_x)], d_y2[(dim_y, dim_x)]]))
    if reg.size == 0:
        reg = np.empty((0, 3))
    bbox = np.transpose(np.vstack([dim_y, dim_x]))
    q_1 = np.fix((stride * bbox + 1) / scale)
    q_2 = np.fix((stride * bbox + cellsize - 1 + 1) / scale)
    boundingbox = np.hstack([q_1, q_2, np.expand_dims(score, 1), reg])
    return boundingbox, reg


# function pick = nms(boxes,threshold,type)
def nms(boxes, threshold, method):
    """ Non_Max Suppression """
    # pylint: disable=too-many-locals
    if boxes.size == 0:
        return np.empty((0, 3))
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = (x_2 - x_1 + 1) * (y_2 - y_1 + 1)
    s_sort = np.argsort(var_s)
    pick = np.zeros_like(var_s, dtype=np.int16)
    counter = 0
    while s_sort.size > 0:
        i = s_sort[-1]
        pick[counter] = i
        counter += 1
        idx = s_sort[0:-1]
        xx_1 = np.maximum(x_1[i], x_1[idx])
        yy_1 = np.maximum(y_1[i], y_1[idx])
        xx_2 = np.minimum(x_2[i], x_2[idx])
        yy_2 = np.minimum(y_2[i], y_2[idx])
        width = np.maximum(0.0, xx_2-xx_1+1)
        height = np.maximum(0.0, yy_2-yy_1+1)
        inter = width * height
        if method == 'Min':
            var_o = inter / np.minimum(area[i], area[idx])
        else:
            var_o = inter / (area[i] + area[idx] - inter)
        s_sort = s_sort[np.where(var_o <= threshold)]
    pick = pick[0:counter]
    return pick


# function [d_y ed_y d_x ed_x y e_y x e_x tmp_width tmp_height] = pad(total_boxes,width,height)
def pad(total_boxes, width, height):
    """Compute the padding coordinates (pad the bounding boxes to square)"""
    tmp_width = (total_boxes[:, 2] - total_boxes[:, 0] + 1).astype(np.int32)
    tmp_height = (total_boxes[:, 3] - total_boxes[:, 1] + 1).astype(np.int32)
    numbox = total_boxes.shape[0]

    d_x = np.ones((numbox), dtype=np.int32)
    d_y = np.ones((numbox), dtype=np.int32)
    ed_x = tmp_width.copy().astype(np.int32)
    ed_y = tmp_height.copy().astype(np.int32)

    dim_x = total_boxes[:, 0].copy().astype(np.int32)
    dim_y = total_boxes[:, 1].copy().astype(np.int32)
    e_x = total_boxes[:, 2].copy().astype(np.int32)
    e_y = total_boxes[:, 3].copy().astype(np.int32)

    tmp = np.where(e_x > width)
    ed_x.flat[tmp] = np.expand_dims(-e_x[tmp] + width + tmp_width[tmp], 1)
    e_x[tmp] = width

    tmp = np.where(e_y > height)
    ed_y.flat[tmp] = np.expand_dims(-e_y[tmp] + height + tmp_height[tmp], 1)
    e_y[tmp] = height

    tmp = np.where(dim_x < 1)
    d_x.flat[tmp] = np.expand_dims(2 - dim_x[tmp], 1)
    dim_x[tmp] = 1

    tmp = np.where(dim_y < 1)
    d_y.flat[tmp] = np.expand_dims(2 - dim_y[tmp], 1)
    dim_y[tmp] = 1

    return d_y, ed_y, d_x, ed_x, dim_y, e_y, dim_x, e_x, tmp_width, tmp_height


# function [bbox_a] = rerec(bbox_a)
def rerec(bbox_a):
    """Convert bbox_a to square."""
    height = bbox_a[:, 3]-bbox_a[:, 1]
    width = bbox_a[:, 2]-bbox_a[:, 0]
    length = np.maximum(width, height)
    bbox_a[:, 0] = bbox_a[:, 0] + width * 0.5 - length * 0.5
    bbox_a[:, 1] = bbox_a[:, 1] + height * 0.5 - length * 0.5
    bbox_a[:, 2:4] = bbox_a[:, 0:2] + np.transpose(np.tile(length, (2, 1)))
    return bbox_a


def imresample(img, size):
    """ Resample image """
    # pylint: disable=no-member
    im_data = cv2.resize(img, (size[1], size[0]),
                         interpolation=cv2.INTER_AREA)  # @UndefinedVariable
    return im_data
 </s> add def rect2square(rectangles):
    """ change rectangles into squares (matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        squares: same as input
    """
    width = rectangles[:, 2] - rectangles[:, 0]
    height = rectangles[:, 3] - rectangles[:, 1]
    length = np.maximum(width, height).T
    rectangles[:, 0] = rectangles[:, 0] + width * 0.5 - length * 0.5
    rectangles[:, 1] = rectangles[:, 1] + height * 0.5 - length * 0.5
    rectangles[:, 2:4] = rectangles[:, 0:2] + np.repeat([length], 2, axis=0).T
    return rectangles </s> remove 
        # Verify that the padding is acceptable
        self.validate_padding(padding)
        # Get the number of channels in the input
        c_i = int(inp.get_shape()[-1])
        # Verify that the grouping parameter is valid
        assert c_i % group == 0
        assert c_o % group == 0
        # Convolution for a given input and kernel
        convolve = lambda i, k: tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding) # noqa
        with tf.variable_scope(name) as scope:
            kernel = self.make_var('weights',
                                   shape=[k_h, k_w, c_i // group, c_o])
            # This is the common-case. Convolve the input without any
            # further complications.
            output = convolve(inp, kernel)
            # Add the biases
            if biased:
                biases = self.make_var('biases', [c_o])
                output = tf.nn.bias_add(output, biases)
            if relu:
                # ReLU non-linearity
                output = tf.nn.relu(output, name=scope.name)
            return output

    @layer
    def prelu(self, inp, name):
        """ Prelu Layer """
        with tf.variable_scope(name):
            i = int(inp.get_shape()[-1])
            alpha = self.make_var('alpha', shape=(i,))
            output = tf.nn.relu(inp) + tf.multiply(alpha, -tf.nn.relu(-inp))
        return output

    @layer
    def max_pool(self, inp, k_h, k_w,  # pylint: disable=too-many-arguments
                 s_h, s_w, name, padding='SAME'):
        """ Max Pool Layer """
        self.validate_padding(padding)
        return tf.nn.max_pool(inp,
                              ksize=[1, k_h, k_w, 1],
                              strides=[1, s_h, s_w, 1],
                              padding=padding,
                              name=name)

    @layer
    def fc(self, inp, num_out, name, relu=True):  # pylint: disable=invalid-name
        """ FC Layer """
        with tf.variable_scope(name):
            input_shape = inp.get_shape()
            if input_shape.ndims == 4:
                # The input is spatial. Vectorize it first.
                dim = 1
                for this_dim in input_shape[1:].as_list():
                    dim *= int(this_dim)
                feed_in = tf.reshape(inp, [-1, dim])
            else:
                feed_in, dim = (inp, input_shape[-1].value)
            weights = self.make_var('weights', shape=[dim, num_out])
            biases = self.make_var('biases', [num_out])
            operator = tf.nn.relu_layer if relu else tf.nn.xw_plus_b
            fc = operator(feed_in, weights, biases, name=name)  # pylint: disable=invalid-name
            return fc

    @layer
    def softmax(self, target, axis, name=None):  # pylint: disable=no-self-use
        """ Multi dimensional softmax,
            refer to https://github.com/tensorflow/tensorflow/issues/210
            compute softmax along the dimension of target
            the native softmax only supports batch_size x dimension """

        max_axis = tf.reduce_max(target, axis, keepdims=True)
        target_exp = tf.exp(target-max_axis)
        normalize = tf.reduce_sum(target_exp, axis, keepdims=True)
        softmax = tf.div(target_exp, normalize, name)
        return softmax


class PNet(Network):
    """ Tensorflow PNet """
    def setup(self):
        (self.feed('data')  # pylint: disable=no-value-for-parameter, no-member
         .conv(3, 3, 10, 1, 1, padding='VALID', relu=False, name='conv1')
         .prelu(name='PReLU1')
         .max_pool(2, 2, 2, 2, name='pool1')
         .conv(3, 3, 16, 1, 1, padding='VALID', relu=False, name='conv2')
         .prelu(name='PReLU2')
         .conv(3, 3, 32, 1, 1, padding='VALID', relu=False, name='conv3')
         .prelu(name='PReLU3')
         .conv(1, 1, 2, 1, 1, relu=False, name='conv4-1')
         .softmax(3, name='prob1'))

        (self.feed('PReLU3')  # pylint: disable=no-value-for-parameter
         .conv(1, 1, 4, 1, 1, relu=False, name='conv4-2'))


class RNet(Network):
    """ Tensorflow RNet """
    def setup(self):
        (self.feed('data')  # pylint: disable=no-value-for-parameter, no-member
         .conv(3, 3, 28, 1, 1, padding='VALID', relu=False, name='conv1')
         .prelu(name='prelu1')
         .max_pool(3, 3, 2, 2, name='pool1')
         .conv(3, 3, 48, 1, 1, padding='VALID', relu=False, name='conv2')
         .prelu(name='prelu2')
         .max_pool(3, 3, 2, 2, padding='VALID', name='pool2')
         .conv(2, 2, 64, 1, 1, padding='VALID', relu=False, name='conv3')
         .prelu(name='prelu3')
         .fc(128, relu=False, name='conv4')
         .prelu(name='prelu4')
         .fc(2, relu=False, name='conv5-1')
         .softmax(1, name='prob1'))

        (self.feed('prelu4')  # pylint: disable=no-value-for-parameter
         .fc(4, relu=False, name='conv5-2'))


class ONet(Network):
    """ Tensorflow ONet """
    def setup(self):
        (self.feed('data')  # pylint: disable=no-value-for-parameter, no-member
         .conv(3, 3, 32, 1, 1, padding='VALID', relu=False, name='conv1')
         .prelu(name='prelu1')
         .max_pool(3, 3, 2, 2, name='pool1')
         .conv(3, 3, 64, 1, 1, padding='VALID', relu=False, name='conv2')
         .prelu(name='prelu2')
         .max_pool(3, 3, 2, 2, padding='VALID', name='pool2')
         .conv(3, 3, 64, 1, 1, padding='VALID', relu=False, name='conv3')
         .prelu(name='prelu3')
         .max_pool(2, 2, 2, 2, name='pool3')
         .conv(2, 2, 128, 1, 1, padding='VALID', relu=False, name='conv4')
         .prelu(name='prelu4')
         .fc(256, relu=False, name='conv5')
         .prelu(name='prelu5')
         .fc(2, relu=False, name='conv6-1')
         .softmax(1, name='prob1'))

        (self.feed('prelu5')  # pylint: disable=no-value-for-parameter
         .fc(4, relu=False, name='conv6-2'))

        (self.feed('prelu5')  # pylint: disable=no-value-for-parameter
         .fc(10, relu=False, name='conv6-3'))


def create_mtcnn(sess, model_path):
    """ Create the network """
    if not model_path:
        model_path, _ = os.path.split(os.path.realpath(__file__))

    with tf.variable_scope('pnet'):
        data = tf.placeholder(tf.float32, (None, None, None, 3), 'input')
        pnet = PNet({'data': data})
        pnet.load(model_path[0], sess)
    with tf.variable_scope('rnet'):
        data = tf.placeholder(tf.float32, (None, 24, 24, 3), 'input')
        rnet = RNet({'data': data})
        rnet.load(model_path[1], sess)
    with tf.variable_scope('onet'):
        data = tf.placeholder(tf.float32, (None, 48, 48, 3), 'input')
        onet = ONet({'data': data})
        onet.load(model_path[2], sess)

    pnet_fun = lambda img: sess.run(('pnet/conv4-2/BiasAdd:0', # noqa
                                     'pnet/prob1:0'),
                                    feed_dict={'pnet/input:0': img})
    rnet_fun = lambda img: sess.run(('rnet/conv5-2/conv5-2:0', # noqa
                                     'rnet/prob1:0'),
                                    feed_dict={'rnet/input:0': img})
    onet_fun = lambda img: sess.run(('onet/conv6-2/conv6-2:0', # noqa
                                     'onet/conv6-3/conv6-3:0',
                                     'onet/prob1:0'),
                                    feed_dict={'onet/input:0': img})
    return pnet_fun, rnet_fun, onet_fun


def detect_face(img, minsize, pnet, rnet,  # pylint: disable=too-many-arguments
                onet, threshold, factor):
    """Detects faces in an image, and returns bounding boxes and points for them.
    img: input image
    minsize: minimum faces' size
    pnet, rnet, onet: caffemodel
    threshold: threshold=[th1, th2, th3], th1-3 are three steps's threshold
    factor: the factor used to create a scaling pyramid of face sizes to
            detect in the image.
 </s> add         """ first stage - fast proposal network (pnet) to obtain face candidates """
        scales = calculate_scales(height, width, self.minsize, self.factor)
        rectangles = []
        for scale in scales:
            scale_img = cv2.resize(image,  # pylint:disable=no-member
                                   (int(width * scale), int(height * scale)))
            input_ = scale_img.reshape(1, *scale_img.shape)
            output = self.pnet.predict(input_)
            # .transpose(0, 2, 1, 3) should be added, but this seems wrong.
            # first 0 select cls score, second 0 = batchnum, alway=0. 1 one hot repr
            cls_prob = output[0][0][:, :, 1]
            roi = output[1][0]
            out_h, out_w = cls_prob.shape
            out_side = max(out_h, out_w)
            cls_prob = np.swapaxes(cls_prob, 0, 1)
            roi = np.swapaxes(roi, 0, 2)
            rectangle = detect_face_12net(cls_prob,
                                          roi,
                                          out_side,
                                          1 / scale,
                                          width,
                                          height,
                                          self.threshold[0])
            rectangles.extend(rectangle)
        return nms(rectangles, 0.7, 'iou')

    def detect_rnet(self, image, rectangles, height, width):
        """ second stage - refinement of face candidates with rnet """
        crop_number = 0
        predict_24_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (24, 24))  # pylint:disable=no-member
            predict_24_batch.append(scale_img)
            crop_number += 1

        predict_24_batch = np.array(predict_24_batch)
        output = self.rnet.predict(predict_24_batch)

        cls_prob = output[0]  # first 0 is to select cls, second batch number, always =0
        cls_prob = np.array(cls_prob)
        roi_prob = output[1]  # first 0 is to select roi, second batch number, always =0
        roi_prob = np.array(roi_prob)
        return filter_face_24net(cls_prob, roi_prob, rectangles, width, height, self.threshold[1])

    def detect_onet(self, image, rectangles, height, width):
        """ third stage - further refinement and facial landmarks positions with onet """
        crop_number = 0
        predict_batch = []
        for rect in rectangles:
            crop_img = image[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])]
            scale_img = cv2.resize(crop_img, (48, 48))  # pylint:disable=no-member
            predict_batch.append(scale_img)
            crop_number += 1

        predict_batch = np.array(predict_batch)

        output = self.onet.predict(predict_batch)
        cls_prob = output[0]
        roi_prob = output[1]
        pts_prob = output[2]  # index
        return filter_face_48net(cls_prob,
                                 roi_prob,
                                 pts_prob,
                                 rectangles,
                                 width,
                                 height,
                                 self.threshold[2])


def detect_face_12net(cls_prob, roi, out_side, scale, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Detect face position and calibrate bounding box on 12net feature map(matrix version)
    Input:
        cls_prob : softmax feature map for face classify
        roi      : feature map for regression
        out_side : feature map's largest size
        scale    : current input image scale in multi-scales
        width    : image's origin width
        height   : image's origin height
        threshold: 0.6 can have 99% recall rate
    """
    in_side = 2*out_side+11
    stride = 0
    if out_side != 1:
        stride = float(in_side-12)/(out_side-1)
    (var_x, var_y) = np.where(cls_prob >= threshold)
    boundingbox = np.array([var_x, var_y]).T
    bb1 = np.fix((stride * (boundingbox) + 0) * scale)
    bb2 = np.fix((stride * (boundingbox) + 11) * scale)
    boundingbox = np.concatenate((bb1, bb2), axis=1)
    dx_1 = roi[0][var_x, var_y]
    dx_2 = roi[1][var_x, var_y]
    dx3 = roi[2][var_x, var_y]
    dx4 = roi[3][var_x, var_y]
    score = np.array([cls_prob[var_x, var_y]]).T
    offset = np.array([dx_1, dx_2, dx3, dx4]).T
    boundingbox = boundingbox + offset*12.0*scale
    rectangles = np.concatenate((boundingbox, score), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, "iou")


def filter_face_24net(cls_prob, roi, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : softmax feature map for face classify
        roi_prob  : feature map for regression
        rectangles: 12net's predict
        width     : image's origin width
        height    : image's origin height
        threshold : 0.6 can have 97% recall rate
    Output:
        rectangles: possible face positions
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_), axis=1)
    rectangles = rect2square(rectangles)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        sc_ = rect[4]
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2, sc_])
    return nms(pick, 0.3, 'iou')


def filter_face_48net(cls_prob, roi, pts, rectangles, width, height, threshold):
    # pylint: disable=too-many-locals, too-many-arguments
    """ Filter face position and calibrate bounding box on 12net's output
    Input:
        cls_prob  : cls_prob[1] is face possibility
        roi       : roi offset
        pts       : 5 landmark
        rectangles: 12net's predict, rectangles[i][0:3] is the position, rectangles[i][4] is score
        width     : image's origin width
        height    : image's origin height
        threshold : 0.7 can have 94% recall rate on CelebA-database
    Output:
        rectangles: face positions and landmarks
    """
    prob = cls_prob[:, 1]
    pick = np.where(prob >= threshold)
    rectangles = np.array(rectangles)
    x_1 = rectangles[pick, 0]
    y_1 = rectangles[pick, 1]
    x_2 = rectangles[pick, 2]
    y_2 = rectangles[pick, 3]
    sc_ = np.array([prob[pick]]).T
    dx_1 = roi[pick, 0]
    dx_2 = roi[pick, 1]
    dx3 = roi[pick, 2]
    dx4 = roi[pick, 3]
    r_width = x_2-x_1
    r_height = y_2-y_1
    pts0 = np.array([(r_width * pts[pick, 0] + x_1)[0]]).T
    pts1 = np.array([(r_height * pts[pick, 5] + y_1)[0]]).T
    pts2 = np.array([(r_width * pts[pick, 1] + x_1)[0]]).T
    pts3 = np.array([(r_height * pts[pick, 6] + y_1)[0]]).T
    pts4 = np.array([(r_width * pts[pick, 2] + x_1)[0]]).T
    pts5 = np.array([(r_height * pts[pick, 7] + y_1)[0]]).T
    pts6 = np.array([(r_width * pts[pick, 3] + x_1)[0]]).T
    pts7 = np.array([(r_height * pts[pick, 8] + y_1)[0]]).T
    pts8 = np.array([(r_width * pts[pick, 4] + x_1)[0]]).T
    pts9 = np.array([(r_height * pts[pick, 9] + y_1)[0]]).T
    x_1 = np.array([(x_1 + dx_1 * r_width)[0]]).T
    y_1 = np.array([(y_1 + dx_2 * r_height)[0]]).T
    x_2 = np.array([(x_2 + dx3 * r_width)[0]]).T
    y_2 = np.array([(y_2 + dx4 * r_height)[0]]).T
    rectangles = np.concatenate((x_1, y_1, x_2, y_2, sc_,
                                 pts0, pts1, pts2, pts3, pts4, pts5, pts6, pts7, pts8, pts9),
                                axis=1)
    pick = []
    for rect in rectangles:
        x_1 = int(max(0, rect[0]))
        y_1 = int(max(0, rect[1]))
        x_2 = int(min(width, rect[2]))
        y_2 = int(min(height, rect[3]))
        if x_2 > x_1 and y_2 > y_1:
            pick.append([x_1, y_1, x_2, y_2,
                         rect[4], rect[5], rect[6], rect[7], rect[8], rect[9],
                         rect[10], rect[11], rect[12], rect[13], rect[14]])
    return nms(pick, 0.3, 'iom')


def nms(rectangles, threshold, method):
    # pylint:disable=too-many-locals
    """ apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if not rectangles:
        return rectangles
    boxes = np.array(rectangles)
    x_1 = boxes[:, 0]
    y_1 = boxes[:, 1]
    x_2 = boxes[:, 2]
    y_2 = boxes[:, 3]
    var_s = boxes[:, 4]
    area = np.multiply(x_2-x_1+1, y_2-y_1+1)
    s_sort = np.array(var_s.argsort())
    pick = []
    while len(s_sort) > 0:
        # s_sort[-1] have hightest prob score, s_sort[0:-1]->others
        xx_1 = np.maximum(x_1[s_sort[-1]], x_1[s_sort[0:-1]])
        yy_1 = np.maximum(y_1[s_sort[-1]], y_1[s_sort[0:-1]])
        xx_2 = np.minimum(x_2[s_sort[-1]], x_2[s_sort[0:-1]])
        yy_2 = np.minimum(y_2[s_sort[-1]], y_2[s_sort[0:-1]])
        width = np.maximum(0.0, xx_2 - xx_1 + 1)
        height = np.maximum(0.0, yy_2 - yy_1 + 1)
        inter = width * height
        if method == 'iom':
            var_o = inter / np.minimum(area[s_sort[-1]], area[s_sort[0:-1]])
        else:
            var_o = inter / (area[s_sort[-1]] + area[s_sort[0:-1]] - inter)
        pick.append(s_sort[-1])
        s_sort = s_sort[np.where(var_o <= threshold)[0]]
    result_rectangle = boxes[pick].tolist()
    return result_rectangle


def calculate_scales(height, width, minsize, factor):
    """ Calculate multi-scale
        Input:
            height: Original image height
            width: Original image width
            minsize: Minimum size for a face to be accepted
            factor: Scaling factor
        Output:
            scales  : Multi-scale </s> remove     def initialize(self, *args, **kwargs):
        """ Calculate batch size """
        super().initialize(*args, **kwargs)
        logger.info("Initializing cv2 DNN Detector...")
        logger.verbose("Using CPU for detection")
        self.detector = cv2.dnn.readNetFromCaffe(self.model_path[1],  # pylint: disable=no-member
                                                 self.model_path[0])
        self.detector.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU)  # pylint: disable=no-member
        self.init.set()
        logger.info("Initialized cv2 DNN Detector.")

    def detect_faces(self, *args, **kwargs):
        """ Detect faces in grayscale image """
        super().detect_faces(*args, **kwargs)
        while True:
            item = self.get_item()
            if item == "EOF":
                break
            logger.trace("Detecting faces: %s", item["filename"])
            [detect_image, scale] = self.compile_detection_image(item["image"],
                                                                 is_square=True,
                                                                 scale_up=True)
            height, width = detect_image.shape[:2]
            for angle in self.rotation:
                current_image, rotmat = self.rotate_image(detect_image, angle)
                logger.trace("Detecting faces")

                blob = cv2.dnn.blobFromImage(current_image,  # pylint: disable=no-member
                                             1.0,
                                             self.target,
                                             [104, 117, 123],
                                             False,
                                             False)
                self.detector.setInput(blob)
                detected = self.detector.forward()
                faces = list()
                for i in range(detected.shape[2]):
                    confidence = detected[0, 0, i, 2]
                    if confidence >= self.confidence:
                        logger.trace("Accepting due to confidence %s >= %s",
                                     confidence, self.confidence)
                        faces.append([(detected[0, 0, i, 3] * width),
                                      (detected[0, 0, i, 4] * height),
                                      (detected[0, 0, i, 5] * width),
                                      (detected[0, 0, i, 6] * height)])

                logger.trace("Detected faces: %s", [face for face in faces])

                if angle != 0 and faces:
                    logger.verbose("found face(s) by rotating image %s degrees", angle)

                if faces:
                    break

            detected_faces = self.process_output(faces, rotmat, scale)
            item["detected_faces"] = detected_faces
            self.finalize(item)

        self.queues["out"].put("EOF")
        logger.debug("Detecting Faces Complete")

    def process_output(self, faces, rotation_matrix, scale):
 </s> add     def init_model(self):
        """ Initialize CV2 DNN Detector Model"""
        self.model = cv2.dnn.readNetFromCaffe(self.model_path[1],  # pylint: disable=no-member
                                              self.model_path[0])
        self.model.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU)  # pylint: disable=no-member

    def process_input(self, batch):
        """ Compile the detection image(s) for prediction """
        batch["feed"] = cv2.dnn.blobFromImages(batch["scaled_image"],  # pylint: disable=no-member
                                               scalefactor=1.0,
                                               size=(self.input_size, self.input_size),
                                               mean=[104, 117, 123],
                                               swapRB=False,
                                               crop=False)
        return batch

    def predict(self, batch):
        """ Run model to get predictions """
        self.model.setInput(batch["feed"])
        predictions = self.model.forward()
        batch["prediction"] = self.finalize_predictions(predictions)
        return batch

    def finalize_predictions(self, predictions):
        """ Filter faces based on confidence level """
        faces = list()
        for i in range(predictions.shape[2]):
            confidence = predictions[0, 0, i, 2]
            if confidence >= self.confidence:
                logger.trace("Accepting due to confidence %s >= %s",
                             confidence, self.confidence)
                faces.append([(predictions[0, 0, i, 3] * self.input_size),
                              (predictions[0, 0, i, 4] * self.input_size),
                              (predictions[0, 0, i, 5] * self.input_size),
                              (predictions[0, 0, i, 6] * self.input_size)])
        logger.trace("faces: %s", faces)
        return [np.array(faces)]

    def process_output(self, batch): </s> remove     def compile_detection_image(self, input_image,  # pylint:disable=too-many-arguments
                                is_square=False, scale_up=False, to_rgb=False,
                                to_grayscale=False, pad_to=None):
        """ Compile the detection image """
        image = input_image.copy()
        if to_rgb:
            image = image[:, :, ::-1]
        elif to_grayscale:
            image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)  # pylint: disable=no-member
        scale = self.set_scale(image, is_square=is_square, scale_up=scale_up)
        image = self.scale_image(image, scale, pad_to)
        if pad_to is None:
            return [image, scale]
        pad_left = int(pad_to[0] - int(input_image.shape[1] * scale)) // 2
        pad_top = int(pad_to[1] - int(input_image.shape[0] * scale)) // 2
        return [image, scale, (pad_left, pad_top)]

    def set_scale(self, image, is_square=False, scale_up=False):
        """ Set the scale factor for incoming image """
        height, width = image.shape[:2]
        if is_square:
            if isinstance(self.target, int):
                dims = (self.target ** 0.5, self.target ** 0.5)
                self.target = dims
            source = max(height, width)
            target = max(self.target)
        else:
            source = (width * height) ** 0.5
            if isinstance(self.target, tuple):
                self.target = self.target[0] * self.target[1]
            target = self.target ** 0.5
 </s> add     def _compile_detection_image(self, input_image):
        """ Compile the detection image for feeding into the model"""
        image = self._convert_color(input_image)
    logger.trace(scales)
    return scales
      while minl >= 12:
          scales += [var_m * np.power(factor, factor_count)]
          minl = minl * factor
          factor_count += 1
    logger.trace(scales)
    return scales
  
  
  def rect2square(rectangles):
      """ change rectangles into squares (matrix version)
      Input:
keep keep keep add keep keep keep keep keep
 De-Multiprocess Extract (#871)

* requirements.txt: - Pin opencv to 4.1.1 (fixes cv2-dnn error)

* lib.face_detect.DetectedFace: change LandmarksXY to landmarks_xy. Add left, right, top, bottom attributes

* lib.model.session: Session manager for loading models into different graphs (for Nvidia + CPU)

* plugins.extract._base: New parent class for all extract plugins

* plugins.extract.pipeline. Remove MultiProcessing. Dynamically limit batchsize for Nvidia cards. Remove loglevel input

* S3FD + FAN plugins. Standardise to Keras version for all backends

* Standardize all extract plugins to new threaded codebase

* Documentation. Start implementing Numpy style docstrings for Sphinx Documentation

* Remove s3fd_amd. Change convert OTF to expect DetectedFace object

* faces_detect - clean up and documentation

* Remove PoolProcess

* Migrate manual tool to new extract workflow

* Remove AMD specific extractor code from cli and plugins

* Sort tool to new extract workflow

* Remove multiprocessing from project

* Remove multiprocessing queues from QueueManager

* Remove multiprocessing support from logger

* Move face_filter to new extraction pipeline

* Alignments landmarksXY > landmarks_xy and legacy handling

* Intercept get_backend for sphinx doc build

# Add Sphinx documentation 
plugins/extract/detect/mtcnn.py