'''
<NAME>
set up :2020-1-9
intergrate img and label into one file

-- fiducial1024_v1
'''


import argparse
import sys, os
import pickle
import random
import collections
import json
import numpy as np
import scipy.io as io
import scipy.misc as m
import matplotlib.pyplot as plt
import glob
import math
import time

import threading
import multiprocessing as mp
from multiprocessing import Pool
import re
import cv2
# sys.path.append('/lustre/home/gwxie/hope/project/dewarp/datasets/')	# /lustre/home/gwxie/program/project/unwarp/perturbed_imgaes/GAN
import utils

def getDatasets(dir):
	return os.listdir(dir)

class perturbed(utils.BasePerturbed):
	def __init__(self, path, bg_path, save_path, save_suffix):

		self.path = path
		self.bg_path = bg_path
		self.save_path = save_path
		self.save_suffix = save_suffix
	def save_img(self, m, n, fold_curve='fold', repeat_time=4, fiducial_points = 16, relativeShift_position='relativeShift_v2'):


		origin_img = cv2.imread(self.path, flags=cv2.IMREAD_COLOR)

		save_img_shape = [512*2, 480*2]		# 320
		# reduce_value = np.random.choice([2**4, 2**5, 2**6, 2**7, 2**8], p=[0.01, 0.1, 0.4, 0.39, 0.1])
		reduce_value = np.random.choice([2*2, 4*2, 8*2, 16*2, 24*2, 32*2, 40*2, 48*2], p=[0.02, 0.18, 0.2, 0.3, 0.1, 0.1, 0.08, 0.02])
		# reduce_value = np.random.choice([8*2, 16*2, 24*2, 32*2, 40*2, 48*2], p=[0.01, 0.02, 0.2, 0.4, 0.19, 0.18])
		# reduce_value = np.random.choice([16, 24, 32, 40, 48, 64], p=[0.01, 0.1, 0.2, 0.4, 0.2, 0.09])
		base_img_shrink = save_img_shape[0] - reduce_value

		# enlarge_img_shrink = [1024, 768]
		# enlarge_img_shrink = [896, 672]		# 420
		enlarge_img_shrink = [512*4, 480*4]	# 420
		# enlarge_img_shrink = [896*2, 768*2]		# 420
		# enlarge_img_shrink = [896, 768]		# 420
		# enlarge_img_shrink = [768, 576]		# 420
		# enlarge_img_shrink = [640, 480]		# 420

		''''''
		im_lr = origin_img.shape[0]
		im_ud = origin_img.shape[1]

		reduce_value_v2 = np.random.choice([2*2, 4*2, 8*2, 16*2, 24*2, 28*2, 32*2, 48*2], p=[0.02, 0.18, 0.2, 0.2, 0.1, 0.1, 0.1, 0.1])
		# reduce_value_v2 = np.random.choice([16, 24, 28, 32, 48, 64], p=[0.01, 0.1, 0.2, 0.3, 0.25, 0.14])

		if im_lr > im_ud:
			im_ud = min(int(im_ud / im_lr * base_img_shrink), save_img_shape[1] - reduce_value_v2)
			im_lr = save_img_shape[0] - reduce_value
		else:
			base_img_shrink = save_img_shape[1] - reduce_value
			im_lr = min(int(im_lr / im_ud * base_img_shrink), save_img_shape[0] - reduce_value_v2)
			im_ud = base_img_shrink

		if round(im_lr / im_ud, 2) < 0.5 or round(im_ud / im_lr, 2) < 0.5:
			repeat_time = min(repeat_time, 8)

		edge_padding = 3
		im_lr -= im_lr % (fiducial_points-1) - (2*edge_padding)		# im_lr % (fiducial_points-1) - 1
		im_ud -= im_ud % (fiducial_points-1) - (2*edge_padding)		# im_ud % (fiducial_points-1) - 1
		im_hight = np.linspace(edge_padding, im_lr - edge_padding, fiducial_points, dtype=np.int64)
		im_wide = np.linspace(edge_padding, im_ud - edge_padding, fiducial_points, dtype=np.int64)
		# im_lr -= im_lr % (fiducial_points-1) - (1+2*edge_padding)		# im_lr % (fiducial_points-1) - 1
		# im_ud -= im_ud % (fiducial_points-1) - (1+2*edge_padding)		# im_ud % (fiducial_points-1) - 1
		# im_hight = np.linspace(edge_padding, im_lr - (1+edge_padding), fiducial_points, dtype=np.int64)
		# im_wide = np.linspace(edge_padding, im_ud - (1+edge_padding), fiducial_points, dtype=np.int64)
		im_x, im_y = np.meshgrid(im_hight, im_wide)
		segment_x = (im_lr) // (fiducial_points-1)
		segment_y = (im_ud) // (fiducial_points-1)

		# plt.plot(im_x, im_y,
		# 		 color='limegreen',
		# 		 marker='.',
		# 		 linestyle='')
		# plt.grid(True)
		# plt.show()
		self.origin_img = cv2.resize(origin_img, (im_ud, im_lr), interpolation=cv2.INTER_CUBIC)

		perturbed_bg_ = getDatasets(self.bg_path)
		perturbed_bg_img_ = self.bg_path+random.choice(perturbed_bg_)
		perturbed_bg_img = cv2.imread(perturbed_bg_img_, flags=cv2.IMREAD_COLOR)

		mesh_shape = self.origin_img.shape[:2]

		self.synthesis_perturbed_img = np.full((enlarge_img_shrink[0], enlarge_img_shrink[1], 3), 256, dtype=np.float32)#np.zeros_like(perturbed_bg_img)
		# self.synthesis_perturbed_img = np.full((enlarge_img_shrink[0], enlarge_img_shrink[1], 3), 0, dtype=np.int16)#np.zeros_like(perturbed_bg_img)
		self.new_shape = self.synthesis_perturbed_img.shape[:2]
		perturbed_bg_img = cv2.resize(perturbed_bg_img, (save_img_shape[1], save_img_shape[0]), cv2.INPAINT_TELEA)

		origin_pixel_position = np.argwhere(np.zeros(mesh_shape, dtype=np.uint32) == 0).reshape(mesh_shape[0], mesh_shape[1], 2)
		pixel_position = np.argwhere(np.zeros(self.new_shape, dtype=np.uint32) == 0).reshape(self.new_shape[0], self.new_shape[1], 2)
		self.perturbed_xy_ = np.zeros((self.new_shape[0], self.new_shape[1], 2))
		# self.perturbed_xy_ = pixel_position.copy().astype(np.float32)
		# fiducial_points_grid = origin_pixel_position[im_x, im_y]

		self.synthesis_perturbed_label = np.zeros((self.new_shape[0], self.new_shape[1], 2))
		x_min, y_min, x_max, y_max = self.adjust_position_v2(0, 0, mesh_shape[0], mesh_shape[1], save_img_shape)
		origin_pixel_position += [x_min, y_min]

		x_min, y_min, x_max, y_max = self.adjust_position(0, 0, mesh_shape[0], mesh_shape[1])
		x_shift = random.randint(-enlarge_img_shrink[0]//16, enlarge_img_shrink[0]//16)
		y_shift = random.randint(-enlarge_img_shrink[1]//16, enlarge_img_shrink[1]//16)
		x_min += x_shift
		x_max += x_shift
		y_min += y_shift
		y_max += y_shift

		'''im_x,y'''
		im_x += x_min
		im_y += y_min

		self.synthesis_perturbed_img[x_min:x_max, y_min:y_max] = self.origin_img
		self.synthesis_perturbed_label[x_min:x_max, y_min:y_max] = origin_pixel_position

		synthesis_perturbed_img_map = self.synthesis_perturbed_img.copy()
		synthesis_perturbed_label_map = self.synthesis_perturbed_label.copy()

		foreORbackground_label = np.full((mesh_shape), 1, dtype=np.int16)
		foreORbackground_label_map = np.full((self.new_shape), 0, dtype=np.int16)
		foreORbackground_label_map[x_min:x_max, y_min:y_max] = foreORbackground_label

		# synthesis_perturbed_img_map = self.pad(self.synthesis_perturbed_img.copy(), x_min, y_min, x_max, y_max)
		# synthesis_perturbed_label_map = self.pad(synthesis_perturbed_label_map, x_min, y_min, x_max, y_max)
		'''*****************************************************************'''
		is_normalizationFun_mixture = self.is_perform(0.2, 0.8)
		# if not is_normalizationFun_mixture:
		normalizationFun_0_1 = False
		# normalizationFun_0_1 = self.is_perform(0.5, 0.5)

		if fold_curve == 'fold':
			fold_curve_random = True
			# is_normalizationFun_mixture = False
			normalizationFun_0_1 = self.is_perform(0.2, 0.8)
			if is_normalizationFun_mixture:
				alpha_perturbed = random.randint(80, 120) / 100
			else:
				if normalizationFun_0_1 and repeat_time < 8:
					alpha_perturbed = random.randint(50, 70) / 100
				else:
					alpha_perturbed = random.randint(70, 130) / 100
		else:
			fold_curve_random = self.is_perform(0.1, 0.9)  # False		# self.is_perform(0.01, 0.99)
			alpha_perturbed = random.randint(80, 160) / 100
			# is_normalizationFun_mixture = False  # self.is_perform(0.01, 0.99)
		synthesis_perturbed_img = np.full_like(self.synthesis_perturbed_img, 256)
		# synthesis_perturbed_img = np.full_like(self.synthesis_perturbed_img, 0, dtype=np.int16)
		synthesis_perturbed_label = np.zeros_like(self.synthesis_perturbed_label)

		alpha_perturbed_change = self.is_perform(0.5, 0.5)
		p_pp_choice = self.is_perform(0.8, 0.2) if fold_curve == 'fold' else self.is_perform(0.1, 0.9)
		for repeat_i in range(repeat_time):

			if alpha_perturbed_change:
				if fold_curve == 'fold':
					if is_normalizationFun_mixture:
						alpha_perturbed = random.randint(80, 120) / 100
					else:
						if normalizationFun_0_1 and repeat_time < 8:
							alpha_perturbed = random.randint(50, 70) / 100
						else:
							alpha_perturbed = random.randint(70, 130) / 100
				else:
					alpha_perturbed = random.randint(80, 160) / 100
			''''''
			linspace_x = [0, (self.new_shape[0] - im_lr) // 2 - 1,
						  self.new_shape[0] - (self.new_shape[0] - im_lr) // 2 - 1, self.new_shape[0] - 1]
			linspace_y = [0, (self.new_shape[1] - im_ud) // 2 - 1,
						  self.new_shape[1] - (self.new_shape[1] - im_ud) // 2 - 1, self.new_shape[1] - 1]
			linspace_x_seq = [1, 2, 3]
			linspace_y_seq = [1, 2, 3]
			r_x = random.choice(linspace_x_seq)
			r_y = random.choice(linspace_y_seq)
			perturbed_p = np.array(
				[random.randint(linspace_x[r_x-1] * 10, linspace_x[r_x] * 10),
				 random.randint(linspace_y[r_y-1] * 10, linspace_y[r_y] * 10)])/10
			if ((r_x == 1 or r_x == 3) and (r_y == 1 or r_y == 3)) and p_pp_choice:
				linspace_x_seq.remove(r_x)
				linspace_y_seq.remove(r_y)
			r_x = random.choice(linspace_x_seq)
			r_y = random.choice(linspace_y_seq)
			perturbed_pp = np.array(
				[random.randint(linspace_x[r_x-1] * 10, linspace_x[r_x] * 10),
				 random.randint(linspace_y[r_y-1] * 10, linspace_y[r_y] * 10)])/10
			# perturbed_p, perturbed_pp = np.array(
			# 	[random.randint(0, self.new_shape[0] * 10) / 10,
			# 	 random.randint(0, self.new_shape[1] * 10) / 10]) \
			# 	, np.array([random.randint(0, self.new_shape[0] * 10) / 10,
			# 	 			random.randint(0, self.new_shape[1] * 10) / 10])
			# perturbed_p, perturbed_pp = np.array(
			# 	[random.randint((self.new_shape[0]-im_lr)//2*10, (self.new_shape[0]-(self.new_shape[0]-im_lr)//2) * 10) / 10,
			# 	 random.randint((self.new_shape[1]-im_ud)//2*10, (self.new_shape[1]-(self.new_shape[1]-im_ud)//2) * 10) / 10]) \
			# 	, np.array([random.randint((self.new_shape[0]-im_lr)//2*10, (self.new_shape[0]-(self.new_shape[0]-im_lr)//2) * 10) / 10,
			# 	 			random.randint((self.new_shape[1]-im_ud)//2*10, (self.new_shape[1]-(self.new_shape[1]-im_ud)//2) * 10) / 10])
			''''''

			perturbed_vp = perturbed_pp - perturbed_p
			perturbed_vp_norm = np.linalg.norm(perturbed_vp)

			perturbed_distance_vertex_and_line = np.dot((perturbed_p - pixel_position), perturbed_vp) / perturbed_vp_norm
			''''''
			# perturbed_v = np.array([random.randint(-3000, 3000) / 100, random.randint(-3000, 3000) / 100])
			# perturbed_v = np.array([random.randint(-4000, 4000) / 100, random.randint(-4000, 4000) / 100])
			if fold_curve == 'fold' and self.is_perform(0.6, 0.4):	# self.is_perform(0.3, 0.7):
				# perturbed_v = np.array([random.randint(-9000, 9000) / 100, random.randint(-9000, 9000) / 100])
				perturbed_v = np.array([random.randint(-10000, 10000) / 100, random.randint(-10000, 10000) / 100])
				# perturbed_v = np.array([random.randint(-11000, 11000) / 100, random.randint(-11000, 11000) / 100])
			else:
				# perturbed_v = np.array([random.randint(-9000, 9000) / 100, random.randint(-9000, 9000) / 100])
				# perturbed_v = np.array([random.randint(-16000, 16000) / 100, random.randint(-16000, 16000) / 100])
				perturbed_v = np.array([random.randint(-8000, 8000) / 100, random.randint(-8000, 8000) / 100])
				# perturbed_v = np.array([random.randint(-3500, 3500) / 100, random.randint(-3500, 3500) / 100])
				# perturbed_v = np.array([random.randint(-600, 600) / 10, random.randint(-600, 600) / 10])
			''''''
			if fold_curve == 'fold':
				if is_normalizationFun_mixture:
					if self.is_perform(0.5, 0.5):
						perturbed_d = np.abs(self.get_normalize(perturbed_distance_vertex_and_line))
					else:
						perturbed_d = self.get_0_1_d(np.abs(perturbed_distance_vertex_and_line), random.randint(1, 2))
				else:
					if normalizationFun_0_1:
						perturbed_d = self.get_0_1_d(np.abs(perturbed_distance_vertex_and_line), 2)
					else:
						perturbed_d = np.abs(self.get_normalize(perturbed_distance_vertex_and_line))

			else:
				if is_normalizationFun_mixture:
					if self.is_perform(0.5, 0.5):
						perturbed_d = np.abs(self.get_normalize(perturbed_distance_vertex_and_line))
					else:
						perturbed_d = self.get_0_1_d(np.abs(perturbed_distance_vertex_and_line), random.randint(1, 2))
				else:
					if normalizationFun_0_1:
						perturbed_d = self.get_0_1_d(np.abs(perturbed_distance_vertex_and_line), 2)
					else:
						perturbed_d = np.abs(self.get_normalize(perturbed_distance_vertex_and_line))
			''''''
			if fold_curve_random:
				# omega_perturbed = (alpha_perturbed+0.2) / (perturbed_d + alpha_perturbed)
				# omega_perturbed = alpha_perturbed**perturbed_d
				omega_perturbed = alpha_perturbed / (perturbed_d + alpha_perturbed)
			else:
				omega_perturbed = 1 - perturbed_d ** alpha_perturbed

			'''shadow'''
			if self.is_perform(0.6, 0.4):
				synthesis_perturbed_img_map[x_min:x_max, y_min:y_max] = np.minimum(np.maximum(synthesis_perturbed_img_map[x_min:x_max, y_min:y_max] - np.int16(np.round(omega_perturbed[x_min:x_max, y_min:y_max].repeat(3).reshape(x_max-x_min, y_max-y_min, 3) * abs(np.linalg.norm(perturbed_v//2))*np.array([0.4-random.random()*0.1, 0.4-random.random()*0.1, 0.4-random.random()*0.1]))), 0), 255)
			''''''

			if relativeShift_position in ['position', 'relativeShift_v2']:
				self.perturbed_xy_ += np.array([omega_perturbed * perturbed_v[0], omega_perturbed * perturbed_v[1]]).transpose(1, 2, 0)
			else:
				print('relativeShift_position error')
				exit()

		'''
		flat_position = np.argwhere(np.zeros(self.new_shape, dtype=np.uint32) == 0).reshape(
			self.new_shape[0] * self.new_shape[1], 2)
		vtx, wts = self.interp_weights(self.perturbed_xy_.reshape(self.new_shape[0] * self.new_shape[1], 2), flat_position)
		wts_sum = np.abs(wts).sum(-1)

		# flat_img.reshape(flat_shape[0] * flat_shape[1], 3)[:] = interpolate(pixel, vtx, wts)
		wts = wts[wts_sum <= 1, :]
		vtx = vtx[wts_sum <= 1, :]
		synthesis_perturbed_img.reshape(self.new_shape[0] * self.new_shape[1], 3)[wts_sum <= 1,
		:] = self.interpolate(synthesis_perturbed_img_map.reshape(self.new_shape[0] * self.new_shape[1], 3), vtx, wts)

		synthesis_perturbed_label.reshape(self.new_shape[0] * self.new_shape[1], 2)[wts_sum <= 1,
		:] = self.interpolate(synthesis_perturbed_label_map.reshape(self.new_shape[0] * self.new_shape[1], 2), vtx, wts)

		foreORbackground_label = np.zeros(self.new_shape)
		foreORbackground_label.reshape(self.new_shape[0] * self.new_shape[1], 1)[wts_sum <= 1, :] = self.interpolate(foreORbackground_label_map.reshape(self.new_shape[0] * self.new_shape[1], 1), vtx, wts)
		foreORbackground_label[foreORbackground_label < 0.99] = 0
		foreORbackground_label[foreORbackground_label >= 0.99] = 1

		# synthesis_perturbed_img = np.around(synthesis_perturbed_img).astype(np.uint8)
		synthesis_perturbed_label[:, :, 0] *= foreORbackground_label
		synthesis_perturbed_label[:, :, 1] *= foreORbackground_label
		synthesis_perturbed_img[:, :, 0] *= foreORbackground_label
		synthesis_perturbed_img[:, :, 1] *= foreORbackground_label
		synthesis_perturbed_img[:, :, 2] *= foreORbackground_label

		self.synthesis_perturbed_img = synthesis_perturbed_img
		self.synthesis_perturbed_label = synthesis_perturbed_label
		'''

		'''perspective'''

		perspective_shreshold = random.randint(26, 36)*10 # 280
		x_min_per, y_min_per, x_max_per, y_max_per = self.adjust_position(perspective_shreshold, perspective_shreshold, self.new_shape[0]-perspective_shreshold, self.new_shape[1]-perspective_shreshold)
		pts1 = np.float32([[x_min_per, y_min_per], [x_max_per, y_min_per], [x_min_per, y_max_per], [x_max_per, y_max_per]])
		e_1_ = x_max_per - x_min_per
		e_2_ = y_max_per - y_min_per
		e_3_ = e_2_
		e_4_ = e_1_
		perspective_shreshold_h = e_1_*0.02
		perspective_shreshold_w = e_2_*0.02
		a_min_, a_max_ = 70, 110
		# if self.is_perform(1, 0):
		if fold_curve == 'curve' and self.is_perform(0.5, 0.5):
			if self.is_perform(0.5, 0.5):
				while True:
					pts2 = np.around(
						np.float32([[x_min_per - (random.random()) * perspective_shreshold, y_min_per + (random.random()) * perspective_shreshold],
									[x_max_per - (random.random()) * perspective_shreshold, y_min_per - (random.random()) * perspective_shreshold],
									[x_min_per + (random.random()) * perspective_shreshold, y_max_per + (random.random()) * perspective_shreshold],
									[x_max_per + (random.random()) * perspective_shreshold, y_max_per - (random.random()) * perspective_shreshold]]))	# right
					e_1 = np.linalg.norm(pts2[0]-pts2[1])
					e_2 = np.linalg.norm(pts2[0]-pts2[2])
					e_3 = np.linalg.norm(pts2[1]-pts2[3])
					e_4 = np.linalg.norm(pts2[2]-pts2[3])
					if e_1_+perspective_shreshold_h > e_1 and e_2_+perspective_shreshold_w > e_2 and e_3_+perspective_shreshold_w > e_3 and e_4_+perspective_shreshold_h > e_4 and \
						e_1_ - perspective_shreshold_h < e_1 and e_2_ - perspective_shreshold_w < e_2 and e_3_ - perspective_shreshold_w < e_3 and e_4_ - perspective_shreshold_h < e_4 and \
						abs(e_1-e_4) < perspective_shreshold_h and abs(e_2-e_3) < perspective_shreshold_w:
						a0_, a1_, a2_, a3_ = self.get_angle_4(pts2)
						if (a0_ > a_min_ and a0_ < a_max_) or (a1_ > a_min_ and a1_ < a_max_) or (a2_ > a_min_ and a2_ < a_max_) or (a3_ > a_min_ and a3_ < a_max_):
							break
			else:
				while True:
					pts2 = np.around(
						np.float32([[x_min_per + (random.random()) * perspective_shreshold, y_min_per - (random.random()) * perspective_shreshold],
									[x_max_per + (random.random()) * perspective_shreshold, y_min_per + (random.random()) * perspective_shreshold],
									[x_min_per - (random.random()) * perspective_shreshold, y_max_per - (random.random()) * perspective_shreshold],
									[x_max_per - (random.random()) * perspective_shreshold, y_max_per + (random.random()) * perspective_shreshold]]))
					e_1 = np.linalg.norm(pts2[0]-pts2[1])
					e_2 = np.linalg.norm(pts2[0]-pts2[2])
					e_3 = np.linalg.norm(pts2[1]-pts2[3])
					e_4 = np.linalg.norm(pts2[2]-pts2[3])
					if e_1_+perspective_shreshold_h > e_1 and e_2_+perspective_shreshold_w > e_2 and e_3_+perspective_shreshold_w > e_3 and e_4_+perspective_shreshold_h > e_4 and \
						e_1_ - perspective_shreshold_h < e_1 and e_2_ - perspective_shreshold_w < e_2 and e_3_ - perspective_shreshold_w < e_3 and e_4_ - perspective_shreshold_h < e_4 and \
						abs(e_1-e_4) < perspective_shreshold_h and abs(e_2-e_3) < perspective_shreshold_w:
						a0_, a1_, a2_, a3_ = self.get_angle_4(pts2)
						if (a0_ > a_min_ and a0_ < a_max_) or (a1_ > a_min_ and a1_ < a_max_) or (a2_ > a_min_ and a2_ < a_max_) or (a3_ > a_min_ and a3_ < a_max_):
							break

		else:
			while True:
				pts2 = np.around(np.float32([[x_min_per+(random.random()-0.5)*perspective_shreshold, y_min_per+(random.random()-0.5)*perspective_shreshold],
								 [x_max_per+(random.random()-0.5)*perspective_shreshold, y_min_per+(random.random()-0.5)*perspective_shreshold],
								 [x_min_per+(random.random()-0.5)*perspective_shreshold, y_max_per+(random.random()-0.5)*perspective_shreshold],
								 [x_max_per+(random.random()-0.5)*perspective_shreshold, y_max_per+(random.random()-0.5)*perspective_shreshold]]))
				e_1 = np.linalg.norm(pts2[0]-pts2[1])
				e_2 = np.linalg.norm(pts2[0]-pts2[2])
				e_3 = np.linalg.norm(pts2[1]-pts2[3])
				e_4 = np.linalg.norm(pts2[2]-pts2[3])
				if e_1_+perspective_shreshold_h > e_1 and e_2_+perspective_shreshold_w > e_2 and e_3_+perspective_shreshold_w > e_3 and e_4_+perspective_shreshold_h > e_4 and \
					e_1_ - perspective_shreshold_h < e_1 and e_2_ - perspective_shreshold_w < e_2 and e_3_ - perspective_shreshold_w < e_3 and e_4_ - perspective_shreshold_h < e_4 and \
					abs(e_1-e_4) < perspective_shreshold_h and abs(e_2-e_3) < perspective_shreshold_w:
					a0_, a1_, a2_, a3_ = self.get_angle_4(pts2)
					if (a0_ > a_min_ and a0_ < a_max_) or (a1_ > a_min_ and a1_ < a_max_) or (a2_ > a_min_ and a2_ < a_max_) or (a3_ > a_min_ and a3_ < a_max_):
						break

		M = cv2.getPerspectiveTransform(pts1, pts2)
		one = np.ones((self.new_shape[0], self.new_shape[1], 1), dtype=np.int16)
		matr = np.dstack((pixel_position, one))
		new = np.dot(M, matr.reshape(-1, 3).T).T.reshape(self.new_shape[0], self.new_shape[1], 3)
		x = new[:, :, 0]/new[:, :, 2]
		y = new[:, :, 1]/new[:, :, 2]
		perturbed_xy_ = np.dstack((x, y))
		# perturbed_xy_round_int = np.around(cv2.bilateralFilter(perturbed_xy_round_int, 9, 75, 75))
		# perturbed_xy_round_int = np.around(cv2.blur(perturbed_xy_, (17, 17)))
		# perturbed_xy_round_int = cv2.blur(perturbed_xy_round_int, (17, 17))
		# perturbed_xy_round_int = cv2.GaussianBlur(perturbed_xy_round_int, (7, 7), 0)
		perturbed_xy_ = perturbed_xy_-np.min(perturbed_xy_.T.reshape(2, -1), 1)
		# perturbed_xy_round_int = np.around(perturbed_xy_round_int-np.min(perturbed_xy_round_int.T.reshape(2, -1), 1)).astype(np.int16)

		self.perturbed_xy_ += perturbed_xy_

		'''perspective end'''

		'''to img'''
		flat_position = np.argwhere(np.zeros(self.new_shape, dtype=np.uint32) == 0).reshape(
			self.new_shape[0] * self.new_shape[1], 2)
		# self.perturbed_xy_ = cv2.blur(self.perturbed_xy_, (7, 7))
		self.perturbed_xy_ = cv2.GaussianBlur(self.perturbed_xy_, (7, 7), 0)

		'''get fiducial points'''
		fiducial_points_coordinate = self.perturbed_xy_[im_x, im_y]


		vtx, wts = self.interp_weights(self.perturbed_xy_.reshape(self.new_shape[0] * self.new_shape[1], 2), flat_position)
		wts_sum = np.abs(wts).sum(-1)

		# flat_img.reshape(flat_shape[0] * flat_shape[1], 3)[:] = interpolate(pixel, vtx, wts)
		wts = wts[wts_sum <= 1, :]
		vtx = vtx[wts_sum <= 1, :]
		synthesis_perturbed_img.reshape(self.new_shape[0] * self.new_shape[1], 3)[wts_sum <= 1,
		:] = self.interpolate(synthesis_perturbed_img_map.reshape(self.new_shape[0] * self.new_shape[1], 3), vtx, wts)

		synthesis_perturbed_label.reshape(self.new_shape[0] * self.new_shape[1], 2)[wts_sum <= 1,
		:] = self.interpolate(synthesis_perturbed_label_map.reshape(self.new_shape[0] * self.new_shape[1], 2), vtx, wts)

		foreORbackground_label = np.zeros(self.new_shape)
		foreORbackground_label.reshape(self.new_shape[0] * self.new_shape[1], 1)[wts_sum <= 1, :] = self.interpolate(foreORbackground_label_map.reshape(self.new_shape[0] * self.new_shape[1], 1), vtx, wts)
		foreORbackground_label[foreORbackground_label < 0.99] = 0
		foreORbackground_label[foreORbackground_label >= 0.99] = 1

		self.synthesis_perturbed_img = synthesis_perturbed_img
		self.synthesis_perturbed_label = synthesis_perturbed_label
		self.foreORbackground_label = foreORbackground_label

		'''draw fiducial points
		stepSize = 0
		fiducial_points_synthesis_perturbed_img = self.synthesis_perturbed_img.copy()
		for l in fiducial_points_coordinate.astype(np.int64).reshape(-1,2):
			cv2.circle(fiducial_points_synthesis_perturbed_img, (l[1] + math.ceil(stepSize / 2), l[0] + math.ceil(stepSize / 2)), 5, (0, 0, 255), -1)
		cv2.imwrite('/lustre/home/gwxie/program/project/unwarp/unwarp_perturbed/TPS/img/cv_TPS_large.jpg', fiducial_points_synthesis_perturbed_img)
		'''

		'''clip'''
		perturbed_x_min, perturbed_y_min, perturbed_x_max, perturbed_y_max = -1, -1, self.new_shape[0], self.new_shape[1]
		for x in range(self.new_shape[0] // 2, perturbed_x_max):
			if np.sum(self.synthesis_perturbed_img[x, :]) == 768 * self.new_shape[1] and perturbed_x_max - 1 > x:
				perturbed_x_max = x
				break
		for x in range(self.new_shape[0] // 2, perturbed_x_min, -1):
			if 
np.sum(self.synthesis_perturbed_img[x, :])