File size: 6,562 Bytes
a6028c9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 |
import cv2
"""
brief: face alignment with FFHQ method (https://github.com/NVlabs/ffhq-dataset)
author: lzhbrian (https://lzhbrian.me)
date: 2020.1.5
note: code is heavily borrowed from
https://github.com/NVlabs/ffhq-dataset
http://dlib.net/face_landmark_detection.py.html
requirements:
apt install cmake
conda install Pillow numpy scipy
pip install dlib
# download face landmark model from:
# http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2
"""
import numpy as np
from PIL import Image
import dlib
class Croper:
def __init__(self, path_of_lm):
# download model from: http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2
self.predictor = dlib.shape_predictor(path_of_lm)
def get_landmark(self, img_np):
"""get landmark with dlib
:return: np.array shape=(68, 2)
"""
detector = dlib.get_frontal_face_detector()
dets = detector(img_np, 1)
# print("Number of faces detected: {}".format(len(dets)))
# for k, d in enumerate(dets):
if len(dets) == 0:
return None
d = dets[0]
# Get the landmarks/parts for the face in box d.
shape = self.predictor(img_np, d)
# print("Part 0: {}, Part 1: {} ...".format(shape.part(0), shape.part(1)))
t = list(shape.parts())
a = []
for tt in t:
a.append([tt.x, tt.y])
lm = np.array(a)
# lm is a shape=(68,2) np.array
return lm
def align_face(self, img, lm, output_size=1024):
"""
:param filepath: str
:return: PIL Image
"""
lm_chin = lm[0: 17] # left-right
lm_eyebrow_left = lm[17: 22] # left-right
lm_eyebrow_right = lm[22: 27] # left-right
lm_nose = lm[27: 31] # top-down
lm_nostrils = lm[31: 36] # top-down
lm_eye_left = lm[36: 42] # left-clockwise
lm_eye_right = lm[42: 48] # left-clockwise
lm_mouth_outer = lm[48: 60] # left-clockwise
lm_mouth_inner = lm[60: 68] # left-clockwise
# Calculate auxiliary vectors.
eye_left = np.mean(lm_eye_left, axis=0)
eye_right = np.mean(lm_eye_right, axis=0)
eye_avg = (eye_left + eye_right) * 0.5
eye_to_eye = eye_right - eye_left
mouth_left = lm_mouth_outer[0]
mouth_right = lm_mouth_outer[6]
mouth_avg = (mouth_left + mouth_right) * 0.5
eye_to_mouth = mouth_avg - eye_avg
# Choose oriented crop rectangle.
x = eye_to_eye - np.flipud(eye_to_mouth) * [-1, 1] # Addition of binocular difference and double mouth difference
x /= np.hypot(*x)
x *= max(np.hypot(*eye_to_eye) * 2.0, np.hypot(*eye_to_mouth) * 1.8)
y = np.flipud(x) * [-1, 1]
c = eye_avg + eye_to_mouth * 0.1
quad = np.stack([c - x - y, c - x + y, c + x + y, c + x - y])
qsize = np.hypot(*x) * 2
# Shrink.
shrink = int(np.floor(qsize / output_size * 0.5))
if shrink > 1:
rsize = (int(np.rint(float(img.size[0]) / shrink)), int(np.rint(float(img.size[1]) / shrink)))
img = img.resize(rsize, Image.ANTIALIAS)
quad /= shrink
qsize /= shrink
else:
rsize = (int(np.rint(float(img.size[0]))), int(np.rint(float(img.size[1]))))
# Crop.
border = max(int(np.rint(qsize * 0.1)), 3)
crop = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))), int(np.ceil(max(quad[:, 0]))),
int(np.ceil(max(quad[:, 1]))))
crop = (max(crop[0] - border, 0), max(crop[1] - border, 0), min(crop[2] + border, img.size[0]),
min(crop[3] + border, img.size[1]))
if crop[2] - crop[0] < img.size[0] or crop[3] - crop[1] < img.size[1]:
# img = img.crop(crop)
quad -= crop[0:2]
# Pad.
pad = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))), int(np.ceil(max(quad[:, 0]))),
int(np.ceil(max(quad[:, 1]))))
pad = (max(-pad[0] + border, 0), max(-pad[1] + border, 0), max(pad[2] - img.size[0] + border, 0),
max(pad[3] - img.size[1] + border, 0))
# if enable_padding and max(pad) > border - 4:
# pad = np.maximum(pad, int(np.rint(qsize * 0.3)))
# img = np.pad(np.float32(img), ((pad[1], pad[3]), (pad[0], pad[2]), (0, 0)), 'reflect')
# h, w, _ = img.shape
# y, x, _ = np.ogrid[:h, :w, :1]
# mask = np.maximum(1.0 - np.minimum(np.float32(x) / pad[0], np.float32(w - 1 - x) / pad[2]),
# 1.0 - np.minimum(np.float32(y) / pad[1], np.float32(h - 1 - y) / pad[3]))
# blur = qsize * 0.02
# img += (scipy.ndimage.gaussian_filter(img, [blur, blur, 0]) - img) * np.clip(mask * 3.0 + 1.0, 0.0, 1.0)
# img += (np.median(img, axis=(0, 1)) - img) * np.clip(mask, 0.0, 1.0)
# img = Image.fromarray(np.uint8(np.clip(np.rint(img), 0, 255)), 'RGB')
# quad += pad[:2]
# Transform.
quad = (quad + 0.5).flatten()
lx = max(min(quad[0], quad[2]), 0)
ly = max(min(quad[1], quad[7]), 0)
rx = min(max(quad[4], quad[6]), img.size[0])
ry = min(max(quad[3], quad[5]), img.size[0])
# img = img.transform((transform_size, transform_size), Image.QUAD, (quad + 0.5).flatten(),
# Image.BILINEAR)
# if output_size < transform_size:
# img = img.resize((output_size, output_size), Image.ANTIALIAS)
# Save aligned image.
return rsize, crop, [lx, ly, rx, ry]
def crop(self, img_np_list, still=False, xsize=512): # first frame for all video
img_np = img_np_list[0]
lm = self.get_landmark(img_np)
if lm is None:
raise 'can not detect the landmark from source image'
rsize, crop, quad = self.align_face(img=Image.fromarray(img_np), lm=lm, output_size=xsize)
clx, cly, crx, cry = crop
lx, ly, rx, ry = quad
lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry)
for _i in range(len(img_np_list)):
_inp = img_np_list[_i]
_inp = cv2.resize(_inp, (rsize[0], rsize[1]))
_inp = _inp[cly:cry, clx:crx]
# cv2.imwrite('test1.jpg', _inp)
if not still:
_inp = _inp[ly:ry, lx:rx]
# cv2.imwrite('test2.jpg', _inp)
img_np_list[_i] = _inp
return img_np_list, crop, quad
|