YoonaAI commited on
Commit
79f1b8f
·
1 Parent(s): 523c2b9

Delete lib

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. lib / pymaf / configs / pymaf_config.yaml +0 -47
  2. lib / pymaf /core / __init__.py +0 -0
  3. lib / pymaf /core / train_options.py +0 -135
  4. lib / pymaf /core /base_trainer.py +0 -107
  5. lib / pymaf /core /cfgs.py +0 -100
  6. lib / pymaf /core /constants.py +0 -153
  7. lib / pymaf /core /fits_dict.py +0 -133
  8. lib / pymaf /core /path_config.py +0 -24
  9. lib / pymaf /models / __init__.py +0 -3
  10. lib / pymaf /models / pymaf_net.py +0 -362
  11. lib / pymaf /models / smpl.py +0 -92
  12. lib / pymaf /models /hmr.py +0 -303
  13. lib / pymaf /models /maf_extractor.py +0 -135
  14. lib / pymaf /models /res_module.py +0 -385
  15. lib / pymaf /utils / __init__.py +0 -0
  16. lib / pymaf /utils / geometry.py +0 -435
  17. lib / pymaf /utils / imutils.py +0 -491
  18. lib / pymaf /utils / streamer.py +0 -142
  19. lib / pymaf /utils /transforms.py +0 -78
  20. lib / renderer / __init__.py +0 -0
  21. lib / renderer / camera.py +0 -226
  22. lib / renderer / gl / __init__.py +0 -0
  23. lib / renderer / gl / data / color.fs +0 -20
  24. lib / renderer / gl / data /color.vs +0 -29
  25. lib / renderer / gl / data /normal.fs +0 -12
  26. lib / renderer / gl / data /normal.vs +0 -15
  27. lib / renderer / gl / data /prt.fs +0 -157
  28. lib / renderer / gl / data /prt.vs +0 -156
  29. lib / renderer / gl / data /prt_uv.fs +0 -141
  30. lib / renderer / gl / data /prt_uv.vs +0 -168
  31. lib / renderer / gl / data /quad.fs +0 -11
  32. lib / renderer / gl / data /quad.vs +0 -11
  33. lib / renderer / gl / framework.py +0 -95
  34. lib / renderer / gl / glcontext.py +0 -136
  35. lib / renderer / gl / init_gl.py +0 -24
  36. lib / renderer / gl / norm_render.py +0 -75
  37. lib / renderer / gl / render.py +0 -380
  38. lib / renderer / gl /cam_render.py +0 -80
  39. lib / renderer / gl /color_render.py +0 -158
  40. lib / renderer / gl /normal_render.py +0 -93
  41. lib / renderer / gl /prt_render.py +0 -450
  42. lib / renderer / gl /render2.py +0 -384
  43. lib / renderer /glm.py +0 -143
  44. lib / renderer /mesh.py +0 -526
  45. lib / renderer /opengl_util.py +0 -369
  46. lib / renderer /prt_util.py +0 -199
  47. lib /common / __init__.py +0 -0
  48. lib /common / seg3d_utils.py +0 -390
  49. lib /common /cloth_extraction.py +0 -170
  50. lib /common /config.py +0 -218
lib / pymaf / configs / pymaf_config.yaml DELETED
@@ -1,47 +0,0 @@
1
- SOLVER:
2
- MAX_ITER: 500000
3
- TYPE: Adam
4
- BASE_LR: 0.00005
5
- GAMMA: 0.1
6
- STEPS: [0]
7
- EPOCHS: [0]
8
- DEBUG: False
9
- LOGDIR: ''
10
- DEVICE: cuda
11
- NUM_WORKERS: 8
12
- SEED_VALUE: -1
13
- LOSS:
14
- KP_2D_W: 300.0
15
- KP_3D_W: 300.0
16
- SHAPE_W: 0.06
17
- POSE_W: 60.0
18
- VERT_W: 0.0
19
- INDEX_WEIGHTS: 2.0
20
- # Loss weights for surface parts. (24 Parts)
21
- PART_WEIGHTS: 0.3
22
- # Loss weights for UV regression.
23
- POINT_REGRESSION_WEIGHTS: 0.5
24
- TRAIN:
25
- NUM_WORKERS: 8
26
- BATCH_SIZE: 64
27
- PIN_MEMORY: True
28
- TEST:
29
- BATCH_SIZE: 32
30
- MODEL:
31
- PyMAF:
32
- BACKBONE: 'res50'
33
- MLP_DIM: [256, 128, 64, 5]
34
- N_ITER: 3
35
- AUX_SUPV_ON: True
36
- DP_HEATMAP_SIZE: 56
37
- RES_MODEL:
38
- DECONV_WITH_BIAS: False
39
- NUM_DECONV_LAYERS: 3
40
- NUM_DECONV_FILTERS:
41
- - 256
42
- - 256
43
- - 256
44
- NUM_DECONV_KERNELS:
45
- - 4
46
- - 4
47
- - 4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /core / __init__.py DELETED
File without changes
lib / pymaf /core / train_options.py DELETED
@@ -1,135 +0,0 @@
1
- import argparse
2
-
3
-
4
- class TrainOptions():
5
- def __init__(self):
6
- self.parser = argparse.ArgumentParser()
7
-
8
- gen = self.parser.add_argument_group('General')
9
- gen.add_argument(
10
- '--resume',
11
- dest='resume',
12
- default=False,
13
- action='store_true',
14
- help='Resume from checkpoint (Use latest checkpoint by default')
15
-
16
- io = self.parser.add_argument_group('io')
17
- io.add_argument('--log_dir',
18
- default='logs',
19
- help='Directory to store logs')
20
- io.add_argument(
21
- '--pretrained_checkpoint',
22
- default=None,
23
- help='Load a pretrained checkpoint at the beginning training')
24
-
25
- train = self.parser.add_argument_group('Training Options')
26
- train.add_argument('--num_epochs',
27
- type=int,
28
- default=200,
29
- help='Total number of training epochs')
30
- train.add_argument('--regressor',
31
- type=str,
32
- choices=['hmr', 'pymaf_net'],
33
- default='pymaf_net',
34
- help='Name of the SMPL regressor.')
35
- train.add_argument('--cfg_file',
36
- type=str,
37
- default='./configs/pymaf_config.yaml',
38
- help='config file path for PyMAF.')
39
- train.add_argument(
40
- '--img_res',
41
- type=int,
42
- default=224,
43
- help='Rescale bounding boxes to size [img_res, img_res] before feeding them in the network'
44
- )
45
- train.add_argument(
46
- '--rot_factor',
47
- type=float,
48
- default=30,
49
- help='Random rotation in the range [-rot_factor, rot_factor]')
50
- train.add_argument(
51
- '--noise_factor',
52
- type=float,
53
- default=0.4,
54
- help='Randomly multiply pixel values with factor in the range [1-noise_factor, 1+noise_factor]'
55
- )
56
- train.add_argument(
57
- '--scale_factor',
58
- type=float,
59
- default=0.25,
60
- help='Rescale bounding boxes by a factor of [1-scale_factor,1+scale_factor]'
61
- )
62
- train.add_argument(
63
- '--openpose_train_weight',
64
- default=0.,
65
- help='Weight for OpenPose keypoints during training')
66
- train.add_argument('--gt_train_weight',
67
- default=1.,
68
- help='Weight for GT keypoints during training')
69
- train.add_argument('--eval_dataset',
70
- type=str,
71
- default='h36m-p2-mosh',
72
- help='Name of the evaluation dataset.')
73
- train.add_argument('--single_dataset',
74
- default=False,
75
- action='store_true',
76
- help='Use a single dataset')
77
- train.add_argument('--single_dataname',
78
- type=str,
79
- default='h36m',
80
- help='Name of the single dataset.')
81
- train.add_argument('--eval_pve',
82
- default=False,
83
- action='store_true',
84
- help='evaluate PVE')
85
- train.add_argument('--overwrite',
86
- default=False,
87
- action='store_true',
88
- help='overwrite the latest checkpoint')
89
-
90
- train.add_argument('--distributed',
91
- action='store_true',
92
- help='Use distributed training')
93
- train.add_argument('--dist_backend',
94
- default='nccl',
95
- type=str,
96
- help='distributed backend')
97
- train.add_argument('--dist_url',
98
- default='tcp://127.0.0.1:10356',
99
- type=str,
100
- help='url used to set up distributed training')
101
- train.add_argument('--world_size',
102
- default=1,
103
- type=int,
104
- help='number of nodes for distributed training')
105
- train.add_argument("--local_rank", default=0, type=int)
106
- train.add_argument('--rank',
107
- default=0,
108
- type=int,
109
- help='node rank for distributed training')
110
- train.add_argument(
111
- '--multiprocessing_distributed',
112
- action='store_true',
113
- help='Use multi-processing distributed training to launch '
114
- 'N processes per node, which has N GPUs. This is the '
115
- 'fastest way to use PyTorch for either single node or '
116
- 'multi node data parallel training')
117
-
118
- misc = self.parser.add_argument_group('Misc Options')
119
- misc.add_argument('--misc',
120
- help="Modify config options using the command-line",
121
- default=None,
122
- nargs=argparse.REMAINDER)
123
- return
124
-
125
- def parse_args(self):
126
- """Parse input arguments."""
127
- self.args = self.parser.parse_args()
128
- self.save_dump()
129
- return self.args
130
-
131
- def save_dump(self):
132
- """Store all argument values to a json file.
133
- The default location is logs/expname/args.json.
134
- """
135
- pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /core /base_trainer.py DELETED
@@ -1,107 +0,0 @@
1
- # This script is borrowed and extended from https://github.com/nkolot/SPIN/blob/master/utils/base_trainer.py
2
- from __future__ import division
3
- import logging
4
- from utils import CheckpointSaver
5
- from tensorboardX import SummaryWriter
6
-
7
- import torch
8
- from tqdm import tqdm
9
-
10
- tqdm.monitor_interval = 0
11
-
12
-
13
- logger = logging.getLogger(__name__)
14
-
15
-
16
- class BaseTrainer(object):
17
- """Base class for Trainer objects.
18
- Takes care of checkpointing/logging/resuming training.
19
- """
20
-
21
- def __init__(self, options):
22
- self.options = options
23
- if options.multiprocessing_distributed:
24
- self.device = torch.device('cuda', options.gpu)
25
- else:
26
- self.device = torch.device(
27
- 'cuda' if torch.cuda.is_available() else 'cpu')
28
- # override this function to define your model, optimizers etc.
29
- self.saver = CheckpointSaver(save_dir=options.checkpoint_dir,
30
- overwrite=options.overwrite)
31
- if options.rank == 0:
32
- self.summary_writer = SummaryWriter(self.options.summary_dir)
33
- self.init_fn()
34
-
35
- self.checkpoint = None
36
- if options.resume and self.saver.exists_checkpoint():
37
- self.checkpoint = self.saver.load_checkpoint(
38
- self.models_dict, self.optimizers_dict)
39
-
40
- if self.checkpoint is None:
41
- self.epoch_count = 0
42
- self.step_count = 0
43
- else:
44
- self.epoch_count = self.checkpoint['epoch']
45
- self.step_count = self.checkpoint['total_step_count']
46
-
47
- if self.checkpoint is not None:
48
- self.checkpoint_batch_idx = self.checkpoint['batch_idx']
49
- else:
50
- self.checkpoint_batch_idx = 0
51
-
52
- self.best_performance = float('inf')
53
-
54
- def load_pretrained(self, checkpoint_file=None):
55
- """Load a pretrained checkpoint.
56
- This is different from resuming training using --resume.
57
- """
58
- if checkpoint_file is not None:
59
- checkpoint = torch.load(checkpoint_file)
60
- for model in self.models_dict:
61
- if model in checkpoint:
62
- self.models_dict[model].load_state_dict(checkpoint[model],
63
- strict=True)
64
- print(f'Checkpoint {model} loaded')
65
-
66
- def move_dict_to_device(self, dict, device, tensor2float=False):
67
- for k, v in dict.items():
68
- if isinstance(v, torch.Tensor):
69
- if tensor2float:
70
- dict[k] = v.float().to(device)
71
- else:
72
- dict[k] = v.to(device)
73
-
74
- # The following methods (with the possible exception of test) have to be implemented in the derived classes
75
- def train(self, epoch):
76
- raise NotImplementedError('You need to provide an train method')
77
-
78
- def init_fn(self):
79
- raise NotImplementedError('You need to provide an _init_fn method')
80
-
81
- def train_step(self, input_batch):
82
- raise NotImplementedError('You need to provide a _train_step method')
83
-
84
- def train_summaries(self, input_batch):
85
- raise NotImplementedError(
86
- 'You need to provide a _train_summaries method')
87
-
88
- def visualize(self, input_batch):
89
- raise NotImplementedError('You need to provide a visualize method')
90
-
91
- def validate(self):
92
- pass
93
-
94
- def test(self):
95
- pass
96
-
97
- def evaluate(self):
98
- pass
99
-
100
- def fit(self):
101
- # Run training for num_epochs epochs
102
- for epoch in tqdm(range(self.epoch_count, self.options.num_epochs),
103
- total=self.options.num_epochs,
104
- initial=self.epoch_count):
105
- self.epoch_count = epoch
106
- self.train(epoch)
107
- return
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /core /cfgs.py DELETED
@@ -1,100 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
-
3
- # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
4
- # holder of all proprietary rights on this computer program.
5
- # You can only use this computer program if you have closed
6
- # a license agreement with MPG or you get the right to use the computer
7
- # program from someone who is authorized to grant you that right.
8
- # Any use of the computer program without a valid license is prohibited and
9
- # liable to prosecution.
10
- #
11
- # Copyright©2019 Max-Planck-Gesellschaft zur Förderung
12
- # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
13
- # for Intelligent Systems. All rights reserved.
14
- #
15
- # Contact: [email protected]
16
-
17
- import os
18
- import json
19
- from yacs.config import CfgNode as CN
20
-
21
- # Configuration variables
22
- cfg = CN(new_allowed=True)
23
-
24
- cfg.OUTPUT_DIR = 'results'
25
- cfg.DEVICE = 'cuda'
26
- cfg.DEBUG = False
27
- cfg.LOGDIR = ''
28
- cfg.VAL_VIS_BATCH_FREQ = 200
29
- cfg.TRAIN_VIS_ITER_FERQ = 1000
30
- cfg.SEED_VALUE = -1
31
-
32
- cfg.TRAIN = CN(new_allowed=True)
33
-
34
- cfg.LOSS = CN(new_allowed=True)
35
- cfg.LOSS.KP_2D_W = 300.0
36
- cfg.LOSS.KP_3D_W = 300.0
37
- cfg.LOSS.SHAPE_W = 0.06
38
- cfg.LOSS.POSE_W = 60.0
39
- cfg.LOSS.VERT_W = 0.0
40
-
41
- # Loss weights for dense correspondences
42
- cfg.LOSS.INDEX_WEIGHTS = 2.0
43
- # Loss weights for surface parts. (24 Parts)
44
- cfg.LOSS.PART_WEIGHTS = 0.3
45
- # Loss weights for UV regression.
46
- cfg.LOSS.POINT_REGRESSION_WEIGHTS = 0.5
47
-
48
- cfg.MODEL = CN(new_allowed=True)
49
-
50
- cfg.MODEL.PyMAF = CN(new_allowed=True)
51
-
52
- # switch
53
- cfg.TRAIN.VAL_LOOP = True
54
-
55
- cfg.TEST = CN(new_allowed=True)
56
-
57
-
58
- def get_cfg_defaults():
59
- """Get a yacs CfgNode object with default values for my_project."""
60
- # Return a clone so that the defaults will not be altered
61
- # This is for the "local variable" use pattern
62
- # return cfg.clone()
63
- return cfg
64
-
65
-
66
- def update_cfg(cfg_file):
67
- # cfg = get_cfg_defaults()
68
- cfg.merge_from_file(cfg_file)
69
- # return cfg.clone()
70
- return cfg
71
-
72
-
73
- def parse_args(args):
74
- cfg_file = args.cfg_file
75
- if args.cfg_file is not None:
76
- cfg = update_cfg(args.cfg_file)
77
- else:
78
- cfg = get_cfg_defaults()
79
-
80
- # if args.misc is not None:
81
- # cfg.merge_from_list(args.misc)
82
-
83
- return cfg
84
-
85
-
86
- def parse_args_extend(args):
87
- if args.resume:
88
- if not os.path.exists(args.log_dir):
89
- raise ValueError(
90
- 'Experiment are set to resume mode, but log directory does not exist.'
91
- )
92
-
93
- # load log's cfg
94
- cfg_file = os.path.join(args.log_dir, 'cfg.yaml')
95
- cfg = update_cfg(cfg_file)
96
-
97
- if args.misc is not None:
98
- cfg.merge_from_list(args.misc)
99
- else:
100
- parse_args(args)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /core /constants.py DELETED
@@ -1,153 +0,0 @@
1
- # This script is borrowed and extended from https://github.com/nkolot/SPIN/blob/master/constants.py
2
- FOCAL_LENGTH = 5000.
3
- IMG_RES = 224
4
-
5
- # Mean and standard deviation for normalizing input image
6
- IMG_NORM_MEAN = [0.485, 0.456, 0.406]
7
- IMG_NORM_STD = [0.229, 0.224, 0.225]
8
- """
9
- We create a superset of joints containing the OpenPose joints together with the ones that each dataset provides.
10
- We keep a superset of 24 joints such that we include all joints from every dataset.
11
- If a dataset doesn't provide annotations for a specific joint, we simply ignore it.
12
- The joints used here are the following:
13
- """
14
- JOINT_NAMES = [
15
- # 25 OpenPose joints (in the order provided by OpenPose)
16
- 'OP Nose',
17
- 'OP Neck',
18
- 'OP RShoulder',
19
- 'OP RElbow',
20
- 'OP RWrist',
21
- 'OP LShoulder',
22
- 'OP LElbow',
23
- 'OP LWrist',
24
- 'OP MidHip',
25
- 'OP RHip',
26
- 'OP RKnee',
27
- 'OP RAnkle',
28
- 'OP LHip',
29
- 'OP LKnee',
30
- 'OP LAnkle',
31
- 'OP REye',
32
- 'OP LEye',
33
- 'OP REar',
34
- 'OP LEar',
35
- 'OP LBigToe',
36
- 'OP LSmallToe',
37
- 'OP LHeel',
38
- 'OP RBigToe',
39
- 'OP RSmallToe',
40
- 'OP RHeel',
41
- # 24 Ground Truth joints (superset of joints from different datasets)
42
- 'Right Ankle',
43
- 'Right Knee',
44
- 'Right Hip', # 2
45
- 'Left Hip',
46
- 'Left Knee', # 4
47
- 'Left Ankle',
48
- 'Right Wrist', # 6
49
- 'Right Elbow',
50
- 'Right Shoulder', # 8
51
- 'Left Shoulder',
52
- 'Left Elbow', # 10
53
- 'Left Wrist',
54
- 'Neck (LSP)', # 12
55
- 'Top of Head (LSP)',
56
- 'Pelvis (MPII)', # 14
57
- 'Thorax (MPII)',
58
- 'Spine (H36M)', # 16
59
- 'Jaw (H36M)',
60
- 'Head (H36M)', # 18
61
- 'Nose',
62
- 'Left Eye',
63
- 'Right Eye',
64
- 'Left Ear',
65
- 'Right Ear'
66
- ]
67
-
68
- # Dict containing the joints in numerical order
69
- JOINT_IDS = {JOINT_NAMES[i]: i for i in range(len(JOINT_NAMES))}
70
-
71
- # Map joints to SMPL joints
72
- JOINT_MAP = {
73
- 'OP Nose': 24,
74
- 'OP Neck': 12,
75
- 'OP RShoulder': 17,
76
- 'OP RElbow': 19,
77
- 'OP RWrist': 21,
78
- 'OP LShoulder': 16,
79
- 'OP LElbow': 18,
80
- 'OP LWrist': 20,
81
- 'OP MidHip': 0,
82
- 'OP RHip': 2,
83
- 'OP RKnee': 5,
84
- 'OP RAnkle': 8,
85
- 'OP LHip': 1,
86
- 'OP LKnee': 4,
87
- 'OP LAnkle': 7,
88
- 'OP REye': 25,
89
- 'OP LEye': 26,
90
- 'OP REar': 27,
91
- 'OP LEar': 28,
92
- 'OP LBigToe': 29,
93
- 'OP LSmallToe': 30,
94
- 'OP LHeel': 31,
95
- 'OP RBigToe': 32,
96
- 'OP RSmallToe': 33,
97
- 'OP RHeel': 34,
98
- 'Right Ankle': 8,
99
- 'Right Knee': 5,
100
- 'Right Hip': 45,
101
- 'Left Hip': 46,
102
- 'Left Knee': 4,
103
- 'Left Ankle': 7,
104
- 'Right Wrist': 21,
105
- 'Right Elbow': 19,
106
- 'Right Shoulder': 17,
107
- 'Left Shoulder': 16,
108
- 'Left Elbow': 18,
109
- 'Left Wrist': 20,
110
- 'Neck (LSP)': 47,
111
- 'Top of Head (LSP)': 48,
112
- 'Pelvis (MPII)': 49,
113
- 'Thorax (MPII)': 50,
114
- 'Spine (H36M)': 51,
115
- 'Jaw (H36M)': 52,
116
- 'Head (H36M)': 53,
117
- 'Nose': 24,
118
- 'Left Eye': 26,
119
- 'Right Eye': 25,
120
- 'Left Ear': 28,
121
- 'Right Ear': 27
122
- }
123
-
124
- # Joint selectors
125
- # Indices to get the 14 LSP joints from the 17 H36M joints
126
- H36M_TO_J17 = [6, 5, 4, 1, 2, 3, 16, 15, 14, 11, 12, 13, 8, 10, 0, 7, 9]
127
- H36M_TO_J14 = H36M_TO_J17[:14]
128
- # Indices to get the 14 LSP joints from the ground truth joints
129
- J24_TO_J17 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 18, 14, 16, 17]
130
- J24_TO_J14 = J24_TO_J17[:14]
131
- J24_TO_J19 = J24_TO_J17[:14] + [19, 20, 21, 22, 23]
132
- J24_TO_JCOCO = [19, 20, 21, 22, 23, 9, 8, 10, 7, 11, 6, 3, 2, 4, 1, 5, 0]
133
-
134
- # Permutation of SMPL pose parameters when flipping the shape
135
- SMPL_JOINTS_FLIP_PERM = [
136
- 0, 2, 1, 3, 5, 4, 6, 8, 7, 9, 11, 10, 12, 14, 13, 15, 17, 16, 19, 18, 21,
137
- 20, 23, 22
138
- ]
139
- SMPL_POSE_FLIP_PERM = []
140
- for i in SMPL_JOINTS_FLIP_PERM:
141
- SMPL_POSE_FLIP_PERM.append(3 * i)
142
- SMPL_POSE_FLIP_PERM.append(3 * i + 1)
143
- SMPL_POSE_FLIP_PERM.append(3 * i + 2)
144
- # Permutation indices for the 24 ground truth joints
145
- J24_FLIP_PERM = [
146
- 5, 4, 3, 2, 1, 0, 11, 10, 9, 8, 7, 6, 12, 13, 14, 15, 16, 17, 18, 19, 21,
147
- 20, 23, 22
148
- ]
149
- # Permutation indices for the full set of 49 joints
150
- J49_FLIP_PERM = [0, 1, 5, 6, 7, 2, 3, 4, 8, 12, 13, 14, 9, 10, 11, 16, 15, 18, 17, 22, 23, 24, 19, 20, 21]\
151
- + [25+i for i in J24_FLIP_PERM]
152
- SMPL_J49_FLIP_PERM = [0, 1, 5, 6, 7, 2, 3, 4, 8, 12, 13, 14, 9, 10, 11, 16, 15, 18, 17, 22, 23, 24, 19, 20, 21]\
153
- + [25+i for i in SMPL_JOINTS_FLIP_PERM]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /core /fits_dict.py DELETED
@@ -1,133 +0,0 @@
1
- '''
2
- This script is borrowed and extended from https://github.com/nkolot/SPIN/blob/master/train/fits_dict.py
3
- '''
4
- import os
5
- import cv2
6
- import torch
7
- import numpy as np
8
- from torchgeometry import angle_axis_to_rotation_matrix, rotation_matrix_to_angle_axis
9
-
10
- from core import path_config, constants
11
-
12
- import logging
13
-
14
- logger = logging.getLogger(__name__)
15
-
16
-
17
- class FitsDict():
18
- """ Dictionary keeping track of the best fit per image in the training set """
19
-
20
- def __init__(self, options, train_dataset):
21
- self.options = options
22
- self.train_dataset = train_dataset
23
- self.fits_dict = {}
24
- self.valid_fit_state = {}
25
- # array used to flip SMPL pose parameters
26
- self.flipped_parts = torch.tensor(constants.SMPL_POSE_FLIP_PERM,
27
- dtype=torch.int64)
28
- # Load dictionary state
29
- for ds_name, ds in train_dataset.dataset_dict.items():
30
- if ds_name in ['h36m']:
31
- dict_file = os.path.join(path_config.FINAL_FITS_DIR,
32
- ds_name + '.npy')
33
- self.fits_dict[ds_name] = torch.from_numpy(np.load(dict_file))
34
- self.valid_fit_state[ds_name] = torch.ones(len(
35
- self.fits_dict[ds_name]),
36
- dtype=torch.uint8)
37
- else:
38
- dict_file = os.path.join(path_config.FINAL_FITS_DIR,
39
- ds_name + '.npz')
40
- fits_dict = np.load(dict_file)
41
- opt_pose = torch.from_numpy(fits_dict['pose'])
42
- opt_betas = torch.from_numpy(fits_dict['betas'])
43
- opt_valid_fit = torch.from_numpy(fits_dict['valid_fit']).to(
44
- torch.uint8)
45
- self.fits_dict[ds_name] = torch.cat([opt_pose, opt_betas],
46
- dim=1)
47
- self.valid_fit_state[ds_name] = opt_valid_fit
48
-
49
- if not options.single_dataset:
50
- for ds in train_dataset.datasets:
51
- if ds.dataset not in ['h36m']:
52
- ds.pose = self.fits_dict[ds.dataset][:, :72].numpy()
53
- ds.betas = self.fits_dict[ds.dataset][:, 72:].numpy()
54
- ds.has_smpl = self.valid_fit_state[ds.dataset].numpy()
55
-
56
- def save(self):
57
- """ Save dictionary state to disk """
58
- for ds_name in self.train_dataset.dataset_dict.keys():
59
- dict_file = os.path.join(self.options.checkpoint_dir,
60
- ds_name + '_fits.npy')
61
- np.save(dict_file, self.fits_dict[ds_name].cpu().numpy())
62
-
63
- def __getitem__(self, x):
64
- """ Retrieve dictionary entries """
65
- dataset_name, ind, rot, is_flipped = x
66
- batch_size = len(dataset_name)
67
- pose = torch.zeros((batch_size, 72))
68
- betas = torch.zeros((batch_size, 10))
69
- for ds, i, n in zip(dataset_name, ind, range(batch_size)):
70
- params = self.fits_dict[ds][i]
71
- pose[n, :] = params[:72]
72
- betas[n, :] = params[72:]
73
- pose = pose.clone()
74
- # Apply flipping and rotation
75
- pose = self.flip_pose(self.rotate_pose(pose, rot), is_flipped)
76
- betas = betas.clone()
77
- return pose, betas
78
-
79
- def get_vaild_state(self, dataset_name, ind):
80
- batch_size = len(dataset_name)
81
- valid_fit = torch.zeros(batch_size, dtype=torch.uint8)
82
- for ds, i, n in zip(dataset_name, ind, range(batch_size)):
83
- valid_fit[n] = self.valid_fit_state[ds][i]
84
- valid_fit = valid_fit.clone()
85
- return valid_fit
86
-
87
- def __setitem__(self, x, val):
88
- """ Update dictionary entries """
89
- dataset_name, ind, rot, is_flipped, update = x
90
- pose, betas = val
91
- batch_size = len(dataset_name)
92
- # Undo flipping and rotation
93
- pose = self.rotate_pose(self.flip_pose(pose, is_flipped), -rot)
94
- params = torch.cat((pose, betas), dim=-1).cpu()
95
- for ds, i, n in zip(dataset_name, ind, range(batch_size)):
96
- if update[n]:
97
- self.fits_dict[ds][i] = params[n]
98
-
99
- def flip_pose(self, pose, is_flipped):
100
- """flip SMPL pose parameters"""
101
- is_flipped = is_flipped.byte()
102
- pose_f = pose.clone()
103
- pose_f[is_flipped, :] = pose[is_flipped][:, self.flipped_parts]
104
- # we also negate the second and the third dimension of the axis-angle representation
105
- pose_f[is_flipped, 1::3] *= -1
106
- pose_f[is_flipped, 2::3] *= -1
107
- return pose_f
108
-
109
- def rotate_pose(self, pose, rot):
110
- """Rotate SMPL pose parameters by rot degrees"""
111
- pose = pose.clone()
112
- cos = torch.cos(-np.pi * rot / 180.)
113
- sin = torch.sin(-np.pi * rot / 180.)
114
- zeros = torch.zeros_like(cos)
115
- r3 = torch.zeros(cos.shape[0], 1, 3, device=cos.device)
116
- r3[:, 0, -1] = 1
117
- R = torch.cat([
118
- torch.stack([cos, -sin, zeros], dim=-1).unsqueeze(1),
119
- torch.stack([sin, cos, zeros], dim=-1).unsqueeze(1), r3
120
- ],
121
- dim=1)
122
- global_pose = pose[:, :3]
123
- global_pose_rotmat = angle_axis_to_rotation_matrix(global_pose)
124
- global_pose_rotmat_3b3 = global_pose_rotmat[:, :3, :3]
125
- global_pose_rotmat_3b3 = torch.matmul(R, global_pose_rotmat_3b3)
126
- global_pose_rotmat[:, :3, :3] = global_pose_rotmat_3b3
127
- global_pose_rotmat = global_pose_rotmat[:, :-1, :-1].cpu().numpy()
128
- global_pose_np = np.zeros((global_pose.shape[0], 3))
129
- for i in range(global_pose.shape[0]):
130
- aa, _ = cv2.Rodrigues(global_pose_rotmat[i])
131
- global_pose_np[i, :] = aa.squeeze()
132
- pose[:, :3] = torch.from_numpy(global_pose_np).to(pose.device)
133
- return pose
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /core /path_config.py DELETED
@@ -1,24 +0,0 @@
1
- """
2
- This script is borrowed and extended from https://github.com/nkolot/SPIN/blob/master/path_config.py
3
- path configuration
4
- This file contains definitions of useful data stuctures and the paths
5
- for the datasets and data files necessary to run the code.
6
- Things you need to change: *_ROOT that indicate the path to each dataset
7
- """
8
- import os
9
- from huggingface_hub import hf_hub_url, cached_download
10
-
11
- # pymaf
12
- pymaf_data_dir = hf_hub_url('Yuliang/PyMAF', '')
13
- smpl_data_dir = hf_hub_url('Yuliang/SMPL', '')
14
- SMPL_MODEL_DIR = os.path.join(smpl_data_dir, 'models/smpl')
15
-
16
- SMPL_MEAN_PARAMS = cached_download(os.path.join(pymaf_data_dir, 'smpl_mean_params.npz'), use_auth_token=os.environ['ICON'])
17
- MESH_DOWNSAMPLEING = cached_download(os.path.join(pymaf_data_dir, 'mesh_downsampling.npz'), use_auth_token=os.environ['ICON'])
18
- CUBE_PARTS_FILE = cached_download(os.path.join(pymaf_data_dir, 'cube_parts.npy'), use_auth_token=os.environ['ICON'])
19
- JOINT_REGRESSOR_TRAIN_EXTRA = cached_download(os.path.join(pymaf_data_dir, 'J_regressor_extra.npy'), use_auth_token=os.environ['ICON'])
20
- JOINT_REGRESSOR_H36M = cached_download(os.path.join(pymaf_data_dir, 'J_regressor_h36m.npy'), use_auth_token=os.environ['ICON'])
21
- VERTEX_TEXTURE_FILE = cached_download(os.path.join(pymaf_data_dir, 'vertex_texture.npy'), use_auth_token=os.environ['ICON'])
22
- SMPL_MEAN_PARAMS = cached_download(os.path.join(pymaf_data_dir, 'smpl_mean_params.npz'), use_auth_token=os.environ['ICON'])
23
- CHECKPOINT_FILE = cached_download(os.path.join(pymaf_data_dir, 'pretrained_model/PyMAF_model_checkpoint.pt'), use_auth_token=os.environ['ICON'])
24
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /models / __init__.py DELETED
@@ -1,3 +0,0 @@
1
- from .hmr import hmr
2
- from .pymaf_net import pymaf_net
3
- from .smpl import SMPL
 
 
 
 
lib / pymaf /models / pymaf_net.py DELETED
@@ -1,362 +0,0 @@
1
- import torch
2
- import torch.nn as nn
3
- import numpy as np
4
-
5
- from lib.pymaf.utils.geometry import rot6d_to_rotmat, projection, rotation_matrix_to_angle_axis
6
- from .maf_extractor import MAF_Extractor
7
- from .smpl import SMPL, SMPL_MODEL_DIR, SMPL_MEAN_PARAMS, H36M_TO_J14
8
- from .hmr import ResNet_Backbone
9
- from .res_module import IUV_predict_layer
10
- from lib.common.config import cfg
11
- import logging
12
-
13
- logger = logging.getLogger(__name__)
14
-
15
- BN_MOMENTUM = 0.1
16
-
17
-
18
- class Regressor(nn.Module):
19
- def __init__(self, feat_dim, smpl_mean_params):
20
- super().__init__()
21
-
22
- npose = 24 * 6
23
-
24
- self.fc1 = nn.Linear(feat_dim + npose + 13, 1024)
25
- self.drop1 = nn.Dropout()
26
- self.fc2 = nn.Linear(1024, 1024)
27
- self.drop2 = nn.Dropout()
28
- self.decpose = nn.Linear(1024, npose)
29
- self.decshape = nn.Linear(1024, 10)
30
- self.deccam = nn.Linear(1024, 3)
31
- nn.init.xavier_uniform_(self.decpose.weight, gain=0.01)
32
- nn.init.xavier_uniform_(self.decshape.weight, gain=0.01)
33
- nn.init.xavier_uniform_(self.deccam.weight, gain=0.01)
34
-
35
- self.smpl = SMPL(SMPL_MODEL_DIR, batch_size=64, create_transl=False)
36
-
37
- mean_params = np.load(smpl_mean_params)
38
- init_pose = torch.from_numpy(mean_params['pose'][:]).unsqueeze(0)
39
- init_shape = torch.from_numpy(
40
- mean_params['shape'][:].astype('float32')).unsqueeze(0)
41
- init_cam = torch.from_numpy(mean_params['cam']).unsqueeze(0)
42
- self.register_buffer('init_pose', init_pose)
43
- self.register_buffer('init_shape', init_shape)
44
- self.register_buffer('init_cam', init_cam)
45
-
46
- def forward(self,
47
- x,
48
- init_pose=None,
49
- init_shape=None,
50
- init_cam=None,
51
- n_iter=1,
52
- J_regressor=None):
53
- batch_size = x.shape[0]
54
-
55
- if init_pose is None:
56
- init_pose = self.init_pose.expand(batch_size, -1)
57
- if init_shape is None:
58
- init_shape = self.init_shape.expand(batch_size, -1)
59
- if init_cam is None:
60
- init_cam = self.init_cam.expand(batch_size, -1)
61
-
62
- pred_pose = init_pose
63
- pred_shape = init_shape
64
- pred_cam = init_cam
65
- for i in range(n_iter):
66
- xc = torch.cat([x, pred_pose, pred_shape, pred_cam], 1)
67
- xc = self.fc1(xc)
68
- xc = self.drop1(xc)
69
- xc = self.fc2(xc)
70
- xc = self.drop2(xc)
71
- pred_pose = self.decpose(xc) + pred_pose
72
- pred_shape = self.decshape(xc) + pred_shape
73
- pred_cam = self.deccam(xc) + pred_cam
74
-
75
- pred_rotmat = rot6d_to_rotmat(pred_pose).view(batch_size, 24, 3, 3)
76
-
77
- pred_output = self.smpl(betas=pred_shape,
78
- body_pose=pred_rotmat[:, 1:],
79
- global_orient=pred_rotmat[:, 0].unsqueeze(1),
80
- pose2rot=False)
81
-
82
- pred_vertices = pred_output.vertices
83
- pred_joints = pred_output.joints
84
- pred_smpl_joints = pred_output.smpl_joints
85
- pred_keypoints_2d = projection(pred_joints, pred_cam)
86
- pose = rotation_matrix_to_angle_axis(pred_rotmat.reshape(-1, 3,
87
- 3)).reshape(
88
- -1, 72)
89
-
90
- if J_regressor is not None:
91
- pred_joints = torch.matmul(J_regressor, pred_vertices)
92
- pred_pelvis = pred_joints[:, [0], :].clone()
93
- pred_joints = pred_joints[:, H36M_TO_J14, :]
94
- pred_joints = pred_joints - pred_pelvis
95
-
96
- output = {
97
- 'theta': torch.cat([pred_cam, pred_shape, pose], dim=1),
98
- 'verts': pred_vertices,
99
- 'kp_2d': pred_keypoints_2d,
100
- 'kp_3d': pred_joints,
101
- 'smpl_kp_3d': pred_smpl_joints,
102
- 'rotmat': pred_rotmat,
103
- 'pred_cam': pred_cam,
104
- 'pred_shape': pred_shape,
105
- 'pred_pose': pred_pose,
106
- }
107
- return output
108
-
109
- def forward_init(self,
110
- x,
111
- init_pose=None,
112
- init_shape=None,
113
- init_cam=None,
114
- n_iter=1,
115
- J_regressor=None):
116
- batch_size = x.shape[0]
117
-
118
- if init_pose is None:
119
- init_pose = self.init_pose.expand(batch_size, -1)
120
- if init_shape is None:
121
- init_shape = self.init_shape.expand(batch_size, -1)
122
- if init_cam is None:
123
- init_cam = self.init_cam.expand(batch_size, -1)
124
-
125
- pred_pose = init_pose
126
- pred_shape = init_shape
127
- pred_cam = init_cam
128
-
129
- pred_rotmat = rot6d_to_rotmat(pred_pose.contiguous()).view(
130
- batch_size, 24, 3, 3)
131
-
132
- pred_output = self.smpl(betas=pred_shape,
133
- body_pose=pred_rotmat[:, 1:],
134
- global_orient=pred_rotmat[:, 0].unsqueeze(1),
135
- pose2rot=False)
136
-
137
- pred_vertices = pred_output.vertices
138
- pred_joints = pred_output.joints
139
- pred_smpl_joints = pred_output.smpl_joints
140
- pred_keypoints_2d = projection(pred_joints, pred_cam)
141
- pose = rotation_matrix_to_angle_axis(pred_rotmat.reshape(-1, 3,
142
- 3)).reshape(
143
- -1, 72)
144
-
145
- if J_regressor is not None:
146
- pred_joints = torch.matmul(J_regressor, pred_vertices)
147
- pred_pelvis = pred_joints[:, [0], :].clone()
148
- pred_joints = pred_joints[:, H36M_TO_J14, :]
149
- pred_joints = pred_joints - pred_pelvis
150
-
151
- output = {
152
- 'theta': torch.cat([pred_cam, pred_shape, pose], dim=1),
153
- 'verts': pred_vertices,
154
- 'kp_2d': pred_keypoints_2d,
155
- 'kp_3d': pred_joints,
156
- 'smpl_kp_3d': pred_smpl_joints,
157
- 'rotmat': pred_rotmat,
158
- 'pred_cam': pred_cam,
159
- 'pred_shape': pred_shape,
160
- 'pred_pose': pred_pose,
161
- }
162
- return output
163
-
164
-
165
- class PyMAF(nn.Module):
166
- """ PyMAF based Deep Regressor for Human Mesh Recovery
167
- PyMAF: 3D Human Pose and Shape Regression with Pyramidal Mesh Alignment Feedback Loop, in ICCV, 2021
168
- """
169
-
170
- def __init__(self, smpl_mean_params=SMPL_MEAN_PARAMS, pretrained=True):
171
- super().__init__()
172
- self.feature_extractor = ResNet_Backbone(
173
- model=cfg.MODEL.PyMAF.BACKBONE, pretrained=pretrained)
174
-
175
- # deconv layers
176
- self.inplanes = self.feature_extractor.inplanes
177
- self.deconv_with_bias = cfg.RES_MODEL.DECONV_WITH_BIAS
178
- self.deconv_layers = self._make_deconv_layer(
179
- cfg.RES_MODEL.NUM_DECONV_LAYERS,
180
- cfg.RES_MODEL.NUM_DECONV_FILTERS,
181
- cfg.RES_MODEL.NUM_DECONV_KERNELS,
182
- )
183
-
184
- self.maf_extractor = nn.ModuleList()
185
- for _ in range(cfg.MODEL.PyMAF.N_ITER):
186
- self.maf_extractor.append(MAF_Extractor())
187
- ma_feat_len = self.maf_extractor[-1].Dmap.shape[
188
- 0] * cfg.MODEL.PyMAF.MLP_DIM[-1]
189
-
190
- grid_size = 21
191
- xv, yv = torch.meshgrid([
192
- torch.linspace(-1, 1, grid_size),
193
- torch.linspace(-1, 1, grid_size)
194
- ])
195
- points_grid = torch.stack([xv.reshape(-1),
196
- yv.reshape(-1)]).unsqueeze(0)
197
- self.register_buffer('points_grid', points_grid)
198
- grid_feat_len = grid_size * grid_size * cfg.MODEL.PyMAF.MLP_DIM[-1]
199
-
200
- self.regressor = nn.ModuleList()
201
- for i in range(cfg.MODEL.PyMAF.N_ITER):
202
- if i == 0:
203
- ref_infeat_dim = grid_feat_len
204
- else:
205
- ref_infeat_dim = ma_feat_len
206
- self.regressor.append(
207
- Regressor(feat_dim=ref_infeat_dim,
208
- smpl_mean_params=smpl_mean_params))
209
-
210
- dp_feat_dim = 256
211
- self.with_uv = cfg.LOSS.POINT_REGRESSION_WEIGHTS > 0
212
- if cfg.MODEL.PyMAF.AUX_SUPV_ON:
213
- self.dp_head = IUV_predict_layer(feat_dim=dp_feat_dim)
214
-
215
- def _make_layer(self, block, planes, blocks, stride=1):
216
- downsample = None
217
- if stride != 1 or self.inplanes != planes * block.expansion:
218
- downsample = nn.Sequential(
219
- nn.Conv2d(self.inplanes,
220
- planes * block.expansion,
221
- kernel_size=1,
222
- stride=stride,
223
- bias=False),
224
- nn.BatchNorm2d(planes * block.expansion),
225
- )
226
-
227
- layers = []
228
- layers.append(block(self.inplanes, planes, stride, downsample))
229
- self.inplanes = planes * block.expansion
230
- for i in range(1, blocks):
231
- layers.append(block(self.inplanes, planes))
232
-
233
- return nn.Sequential(*layers)
234
-
235
- def _make_deconv_layer(self, num_layers, num_filters, num_kernels):
236
- """
237
- Deconv_layer used in Simple Baselines:
238
- Xiao et al. Simple Baselines for Human Pose Estimation and Tracking
239
- https://github.com/microsoft/human-pose-estimation.pytorch
240
- """
241
- assert num_layers == len(num_filters), \
242
- 'ERROR: num_deconv_layers is different len(num_deconv_filters)'
243
- assert num_layers == len(num_kernels), \
244
- 'ERROR: num_deconv_layers is different len(num_deconv_filters)'
245
-
246
- def _get_deconv_cfg(deconv_kernel, index):
247
- if deconv_kernel == 4:
248
- padding = 1
249
- output_padding = 0
250
- elif deconv_kernel == 3:
251
- padding = 1
252
- output_padding = 1
253
- elif deconv_kernel == 2:
254
- padding = 0
255
- output_padding = 0
256
-
257
- return deconv_kernel, padding, output_padding
258
-
259
- layers = []
260
- for i in range(num_layers):
261
- kernel, padding, output_padding = _get_deconv_cfg(
262
- num_kernels[i], i)
263
-
264
- planes = num_filters[i]
265
- layers.append(
266
- nn.ConvTranspose2d(in_channels=self.inplanes,
267
- out_channels=planes,
268
- kernel_size=kernel,
269
- stride=2,
270
- padding=padding,
271
- output_padding=output_padding,
272
- bias=self.deconv_with_bias))
273
- layers.append(nn.BatchNorm2d(planes, momentum=BN_MOMENTUM))
274
- layers.append(nn.ReLU(inplace=True))
275
- self.inplanes = planes
276
-
277
- return nn.Sequential(*layers)
278
-
279
- def forward(self, x, J_regressor=None):
280
-
281
- batch_size = x.shape[0]
282
-
283
- # spatial features and global features
284
- s_feat, g_feat = self.feature_extractor(x)
285
-
286
- assert cfg.MODEL.PyMAF.N_ITER >= 0 and cfg.MODEL.PyMAF.N_ITER <= 3
287
- if cfg.MODEL.PyMAF.N_ITER == 1:
288
- deconv_blocks = [self.deconv_layers]
289
- elif cfg.MODEL.PyMAF.N_ITER == 2:
290
- deconv_blocks = [self.deconv_layers[0:6], self.deconv_layers[6:9]]
291
- elif cfg.MODEL.PyMAF.N_ITER == 3:
292
- deconv_blocks = [
293
- self.deconv_layers[0:3], self.deconv_layers[3:6],
294
- self.deconv_layers[6:9]
295
- ]
296
-
297
- out_list = {}
298
-
299
- # initial parameters
300
- # TODO: remove the initial mesh generation during forward to reduce runtime
301
- # by generating initial mesh the beforehand: smpl_output = self.init_smpl
302
- smpl_output = self.regressor[0].forward_init(g_feat,
303
- J_regressor=J_regressor)
304
-
305
- out_list['smpl_out'] = [smpl_output]
306
- out_list['dp_out'] = []
307
-
308
- # for visulization
309
- vis_feat_list = [s_feat.detach()]
310
-
311
- # parameter predictions
312
- for rf_i in range(cfg.MODEL.PyMAF.N_ITER):
313
- pred_cam = smpl_output['pred_cam']
314
- pred_shape = smpl_output['pred_shape']
315
- pred_pose = smpl_output['pred_pose']
316
-
317
- pred_cam = pred_cam.detach()
318
- pred_shape = pred_shape.detach()
319
- pred_pose = pred_pose.detach()
320
-
321
- s_feat_i = deconv_blocks[rf_i](s_feat)
322
- s_feat = s_feat_i
323
- vis_feat_list.append(s_feat_i.detach())
324
-
325
- self.maf_extractor[rf_i].im_feat = s_feat_i
326
- self.maf_extractor[rf_i].cam = pred_cam
327
-
328
- if rf_i == 0:
329
- sample_points = torch.transpose(
330
- self.points_grid.expand(batch_size, -1, -1), 1, 2)
331
- ref_feature = self.maf_extractor[rf_i].sampling(sample_points)
332
- else:
333
- pred_smpl_verts = smpl_output['verts'].detach()
334
- # TODO: use a more sparse SMPL implementation (with 431 vertices) for acceleration
335
- pred_smpl_verts_ds = torch.matmul(
336
- self.maf_extractor[rf_i].Dmap.unsqueeze(0),
337
- pred_smpl_verts) # [B, 431, 3]
338
- ref_feature = self.maf_extractor[rf_i](
339
- pred_smpl_verts_ds) # [B, 431 * n_feat]
340
-
341
- smpl_output = self.regressor[rf_i](ref_feature,
342
- pred_pose,
343
- pred_shape,
344
- pred_cam,
345
- n_iter=1,
346
- J_regressor=J_regressor)
347
- out_list['smpl_out'].append(smpl_output)
348
-
349
- if self.training and cfg.MODEL.PyMAF.AUX_SUPV_ON:
350
- iuv_out_dict = self.dp_head(s_feat)
351
- out_list['dp_out'].append(iuv_out_dict)
352
-
353
- return out_list
354
-
355
-
356
- def pymaf_net(smpl_mean_params, pretrained=True):
357
- """ Constructs an PyMAF model with ResNet50 backbone.
358
- Args:
359
- pretrained (bool): If True, returns a model pre-trained on ImageNet
360
- """
361
- model = PyMAF(smpl_mean_params, pretrained)
362
- return model
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /models / smpl.py DELETED
@@ -1,92 +0,0 @@
1
- # This script is borrowed from https://github.com/nkolot/SPIN/blob/master/models/smpl.py
2
-
3
- import torch
4
- import numpy as np
5
- from lib.smplx import SMPL as _SMPL
6
- from lib.smplx.body_models import ModelOutput
7
- from lib.smplx.lbs import vertices2joints
8
- from collections import namedtuple
9
-
10
- from lib.pymaf.core import path_config, constants
11
-
12
- SMPL_MEAN_PARAMS = path_config.SMPL_MEAN_PARAMS
13
- SMPL_MODEL_DIR = path_config.SMPL_MODEL_DIR
14
-
15
- # Indices to get the 14 LSP joints from the 17 H36M joints
16
- H36M_TO_J17 = [6, 5, 4, 1, 2, 3, 16, 15, 14, 11, 12, 13, 8, 10, 0, 7, 9]
17
- H36M_TO_J14 = H36M_TO_J17[:14]
18
-
19
-
20
- class SMPL(_SMPL):
21
- """ Extension of the official SMPL implementation to support more joints """
22
-
23
- def __init__(self, *args, **kwargs):
24
- super().__init__(*args, **kwargs)
25
- joints = [constants.JOINT_MAP[i] for i in constants.JOINT_NAMES]
26
- J_regressor_extra = np.load(path_config.JOINT_REGRESSOR_TRAIN_EXTRA)
27
- self.register_buffer(
28
- 'J_regressor_extra',
29
- torch.tensor(J_regressor_extra, dtype=torch.float32))
30
- self.joint_map = torch.tensor(joints, dtype=torch.long)
31
- self.ModelOutput = namedtuple(
32
- 'ModelOutput_', ModelOutput._fields + (
33
- 'smpl_joints',
34
- 'joints_J19',
35
- ))
36
- self.ModelOutput.__new__.__defaults__ = (None, ) * len(
37
- self.ModelOutput._fields)
38
-
39
- def forward(self, *args, **kwargs):
40
- kwargs['get_skin'] = True
41
- smpl_output = super().forward(*args, **kwargs)
42
- extra_joints = vertices2joints(self.J_regressor_extra,
43
- smpl_output.vertices)
44
- # smpl_output.joints: [B, 45, 3] extra_joints: [B, 9, 3]
45
- vertices = smpl_output.vertices
46
- joints = torch.cat([smpl_output.joints, extra_joints], dim=1)
47
- smpl_joints = smpl_output.joints[:, :24]
48
- joints = joints[:, self.joint_map, :] # [B, 49, 3]
49
- joints_J24 = joints[:, -24:, :]
50
- joints_J19 = joints_J24[:, constants.J24_TO_J19, :]
51
- output = self.ModelOutput(vertices=vertices,
52
- global_orient=smpl_output.global_orient,
53
- body_pose=smpl_output.body_pose,
54
- joints=joints,
55
- joints_J19=joints_J19,
56
- smpl_joints=smpl_joints,
57
- betas=smpl_output.betas,
58
- full_pose=smpl_output.full_pose)
59
- return output
60
-
61
-
62
- def get_smpl_faces():
63
- smpl = SMPL(SMPL_MODEL_DIR, batch_size=1, create_transl=False)
64
- return smpl.faces
65
-
66
-
67
- def get_part_joints(smpl_joints):
68
- batch_size = smpl_joints.shape[0]
69
-
70
- # part_joints = torch.zeros().to(smpl_joints.device)
71
-
72
- one_seg_pairs = [(0, 1), (0, 2), (0, 3), (3, 6), (9, 12), (9, 13), (9, 14),
73
- (12, 15), (13, 16), (14, 17)]
74
- two_seg_pairs = [(1, 4), (2, 5), (4, 7), (5, 8), (16, 18), (17, 19),
75
- (18, 20), (19, 21)]
76
-
77
- one_seg_pairs.extend(two_seg_pairs)
78
-
79
- single_joints = [(10), (11), (15), (22), (23)]
80
-
81
- part_joints = []
82
-
83
- for j_p in one_seg_pairs:
84
- new_joint = torch.mean(smpl_joints[:, j_p], dim=1, keepdim=True)
85
- part_joints.append(new_joint)
86
-
87
- for j_p in single_joints:
88
- part_joints.append(smpl_joints[:, j_p:j_p + 1])
89
-
90
- part_joints = torch.cat(part_joints, dim=1)
91
-
92
- return part_joints
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /models /hmr.py DELETED
@@ -1,303 +0,0 @@
1
- # This script is borrowed from https://github.com/nkolot/SPIN/blob/master/models/hmr.py
2
-
3
- import torch
4
- import torch.nn as nn
5
- import torchvision.models.resnet as resnet
6
- import numpy as np
7
- import math
8
- from lib.pymaf.utils.geometry import rot6d_to_rotmat
9
-
10
- import logging
11
-
12
- logger = logging.getLogger(__name__)
13
-
14
- BN_MOMENTUM = 0.1
15
-
16
-
17
- class Bottleneck(nn.Module):
18
- """ Redefinition of Bottleneck residual block
19
- Adapted from the official PyTorch implementation
20
- """
21
- expansion = 4
22
-
23
- def __init__(self, inplanes, planes, stride=1, downsample=None):
24
- super().__init__()
25
- self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
26
- self.bn1 = nn.BatchNorm2d(planes)
27
- self.conv2 = nn.Conv2d(planes,
28
- planes,
29
- kernel_size=3,
30
- stride=stride,
31
- padding=1,
32
- bias=False)
33
- self.bn2 = nn.BatchNorm2d(planes)
34
- self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
35
- self.bn3 = nn.BatchNorm2d(planes * 4)
36
- self.relu = nn.ReLU(inplace=True)
37
- self.downsample = downsample
38
- self.stride = stride
39
-
40
- def forward(self, x):
41
- residual = x
42
-
43
- out = self.conv1(x)
44
- out = self.bn1(out)
45
- out = self.relu(out)
46
-
47
- out = self.conv2(out)
48
- out = self.bn2(out)
49
- out = self.relu(out)
50
-
51
- out = self.conv3(out)
52
- out = self.bn3(out)
53
-
54
- if self.downsample is not None:
55
- residual = self.downsample(x)
56
-
57
- out += residual
58
- out = self.relu(out)
59
-
60
- return out
61
-
62
-
63
- class ResNet_Backbone(nn.Module):
64
- """ Feature Extrator with ResNet backbone
65
- """
66
-
67
- def __init__(self, model='res50', pretrained=True):
68
- if model == 'res50':
69
- block, layers = Bottleneck, [3, 4, 6, 3]
70
- else:
71
- pass # TODO
72
-
73
- self.inplanes = 64
74
- super().__init__()
75
- npose = 24 * 6
76
- self.conv1 = nn.Conv2d(3,
77
- 64,
78
- kernel_size=7,
79
- stride=2,
80
- padding=3,
81
- bias=False)
82
- self.bn1 = nn.BatchNorm2d(64)
83
- self.relu = nn.ReLU(inplace=True)
84
- self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
85
- self.layer1 = self._make_layer(block, 64, layers[0])
86
- self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
87
- self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
88
- self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
89
- self.avgpool = nn.AvgPool2d(7, stride=1)
90
-
91
- if pretrained:
92
- resnet_imagenet = resnet.resnet50(pretrained=True)
93
- self.load_state_dict(resnet_imagenet.state_dict(), strict=False)
94
- logger.info('loaded resnet50 imagenet pretrained model')
95
-
96
- def _make_layer(self, block, planes, blocks, stride=1):
97
- downsample = None
98
- if stride != 1 or self.inplanes != planes * block.expansion:
99
- downsample = nn.Sequential(
100
- nn.Conv2d(self.inplanes,
101
- planes * block.expansion,
102
- kernel_size=1,
103
- stride=stride,
104
- bias=False),
105
- nn.BatchNorm2d(planes * block.expansion),
106
- )
107
-
108
- layers = []
109
- layers.append(block(self.inplanes, planes, stride, downsample))
110
- self.inplanes = planes * block.expansion
111
- for i in range(1, blocks):
112
- layers.append(block(self.inplanes, planes))
113
-
114
- return nn.Sequential(*layers)
115
-
116
- def _make_deconv_layer(self, num_layers, num_filters, num_kernels):
117
- assert num_layers == len(num_filters), \
118
- 'ERROR: num_deconv_layers is different len(num_deconv_filters)'
119
- assert num_layers == len(num_kernels), \
120
- 'ERROR: num_deconv_layers is different len(num_deconv_filters)'
121
-
122
- def _get_deconv_cfg(deconv_kernel, index):
123
- if deconv_kernel == 4:
124
- padding = 1
125
- output_padding = 0
126
- elif deconv_kernel == 3:
127
- padding = 1
128
- output_padding = 1
129
- elif deconv_kernel == 2:
130
- padding = 0
131
- output_padding = 0
132
-
133
- return deconv_kernel, padding, output_padding
134
-
135
- layers = []
136
- for i in range(num_layers):
137
- kernel, padding, output_padding = _get_deconv_cfg(
138
- num_kernels[i], i)
139
-
140
- planes = num_filters[i]
141
- layers.append(
142
- nn.ConvTranspose2d(in_channels=self.inplanes,
143
- out_channels=planes,
144
- kernel_size=kernel,
145
- stride=2,
146
- padding=padding,
147
- output_padding=output_padding,
148
- bias=self.deconv_with_bias))
149
- layers.append(nn.BatchNorm2d(planes, momentum=BN_MOMENTUM))
150
- layers.append(nn.ReLU(inplace=True))
151
- self.inplanes = planes
152
-
153
- return nn.Sequential(*layers)
154
-
155
- def forward(self, x):
156
-
157
- batch_size = x.shape[0]
158
-
159
- x = self.conv1(x)
160
- x = self.bn1(x)
161
- x = self.relu(x)
162
- x = self.maxpool(x)
163
-
164
- x1 = self.layer1(x)
165
- x2 = self.layer2(x1)
166
- x3 = self.layer3(x2)
167
- x4 = self.layer4(x3)
168
-
169
- xf = self.avgpool(x4)
170
- xf = xf.view(xf.size(0), -1)
171
-
172
- x_featmap = x4
173
-
174
- return x_featmap, xf
175
-
176
-
177
- class HMR(nn.Module):
178
- """ SMPL Iterative Regressor with ResNet50 backbone
179
- """
180
-
181
- def __init__(self, block, layers, smpl_mean_params):
182
- self.inplanes = 64
183
- super().__init__()
184
- npose = 24 * 6
185
- self.conv1 = nn.Conv2d(3,
186
- 64,
187
- kernel_size=7,
188
- stride=2,
189
- padding=3,
190
- bias=False)
191
- self.bn1 = nn.BatchNorm2d(64)
192
- self.relu = nn.ReLU(inplace=True)
193
- self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
194
- self.layer1 = self._make_layer(block, 64, layers[0])
195
- self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
196
- self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
197
- self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
198
- self.avgpool = nn.AvgPool2d(7, stride=1)
199
- self.fc1 = nn.Linear(512 * block.expansion + npose + 13, 1024)
200
- self.drop1 = nn.Dropout()
201
- self.fc2 = nn.Linear(1024, 1024)
202
- self.drop2 = nn.Dropout()
203
- self.decpose = nn.Linear(1024, npose)
204
- self.decshape = nn.Linear(1024, 10)
205
- self.deccam = nn.Linear(1024, 3)
206
- nn.init.xavier_uniform_(self.decpose.weight, gain=0.01)
207
- nn.init.xavier_uniform_(self.decshape.weight, gain=0.01)
208
- nn.init.xavier_uniform_(self.deccam.weight, gain=0.01)
209
-
210
- for m in self.modules():
211
- if isinstance(m, nn.Conv2d):
212
- n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
213
- m.weight.data.normal_(0, math.sqrt(2. / n))
214
- elif isinstance(m, nn.BatchNorm2d):
215
- m.weight.data.fill_(1)
216
- m.bias.data.zero_()
217
-
218
- mean_params = np.load(smpl_mean_params)
219
- init_pose = torch.from_numpy(mean_params['pose'][:]).unsqueeze(0)
220
- init_shape = torch.from_numpy(
221
- mean_params['shape'][:].astype('float32')).unsqueeze(0)
222
- init_cam = torch.from_numpy(mean_params['cam']).unsqueeze(0)
223
- self.register_buffer('init_pose', init_pose)
224
- self.register_buffer('init_shape', init_shape)
225
- self.register_buffer('init_cam', init_cam)
226
-
227
- def _make_layer(self, block, planes, blocks, stride=1):
228
- downsample = None
229
- if stride != 1 or self.inplanes != planes * block.expansion:
230
- downsample = nn.Sequential(
231
- nn.Conv2d(self.inplanes,
232
- planes * block.expansion,
233
- kernel_size=1,
234
- stride=stride,
235
- bias=False),
236
- nn.BatchNorm2d(planes * block.expansion),
237
- )
238
-
239
- layers = []
240
- layers.append(block(self.inplanes, planes, stride, downsample))
241
- self.inplanes = planes * block.expansion
242
- for i in range(1, blocks):
243
- layers.append(block(self.inplanes, planes))
244
-
245
- return nn.Sequential(*layers)
246
-
247
- def forward(self,
248
- x,
249
- init_pose=None,
250
- init_shape=None,
251
- init_cam=None,
252
- n_iter=3):
253
-
254
- batch_size = x.shape[0]
255
-
256
- if init_pose is None:
257
- init_pose = self.init_pose.expand(batch_size, -1)
258
- if init_shape is None:
259
- init_shape = self.init_shape.expand(batch_size, -1)
260
- if init_cam is None:
261
- init_cam = self.init_cam.expand(batch_size, -1)
262
-
263
- x = self.conv1(x)
264
- x = self.bn1(x)
265
- x = self.relu(x)
266
- x = self.maxpool(x)
267
-
268
- x1 = self.layer1(x)
269
- x2 = self.layer2(x1)
270
- x3 = self.layer3(x2)
271
- x4 = self.layer4(x3)
272
-
273
- xf = self.avgpool(x4)
274
- xf = xf.view(xf.size(0), -1)
275
-
276
- pred_pose = init_pose
277
- pred_shape = init_shape
278
- pred_cam = init_cam
279
- for i in range(n_iter):
280
- xc = torch.cat([xf, pred_pose, pred_shape, pred_cam], 1)
281
- xc = self.fc1(xc)
282
- xc = self.drop1(xc)
283
- xc = self.fc2(xc)
284
- xc = self.drop2(xc)
285
- pred_pose = self.decpose(xc) + pred_pose
286
- pred_shape = self.decshape(xc) + pred_shape
287
- pred_cam = self.deccam(xc) + pred_cam
288
-
289
- pred_rotmat = rot6d_to_rotmat(pred_pose).view(batch_size, 24, 3, 3)
290
-
291
- return pred_rotmat, pred_shape, pred_cam
292
-
293
-
294
- def hmr(smpl_mean_params, pretrained=True, **kwargs):
295
- """ Constructs an HMR model with ResNet50 backbone.
296
- Args:
297
- pretrained (bool): If True, returns a model pre-trained on ImageNet
298
- """
299
- model = HMR(Bottleneck, [3, 4, 6, 3], smpl_mean_params, **kwargs)
300
- if pretrained:
301
- resnet_imagenet = resnet.resnet50(pretrained=True)
302
- model.load_state_dict(resnet_imagenet.state_dict(), strict=False)
303
- return model
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /models /maf_extractor.py DELETED
@@ -1,135 +0,0 @@
1
- # This script is borrowed and extended from https://github.com/shunsukesaito/PIFu/blob/master/lib/model/SurfaceClassifier.py
2
-
3
- from packaging import version
4
- import torch
5
- import scipy
6
- import numpy as np
7
- import torch.nn as nn
8
- import torch.nn.functional as F
9
-
10
- from lib.common.config import cfg
11
- from lib.pymaf.utils.geometry import projection
12
- from lib.pymaf.core.path_config import MESH_DOWNSAMPLEING
13
-
14
- import logging
15
-
16
- logger = logging.getLogger(__name__)
17
-
18
-
19
- class MAF_Extractor(nn.Module):
20
- ''' Mesh-aligned Feature Extrator
21
- As discussed in the paper, we extract mesh-aligned features based on 2D projection of the mesh vertices.
22
- The features extrated from spatial feature maps will go through a MLP for dimension reduction.
23
- '''
24
-
25
- def __init__(self, device=torch.device('cuda')):
26
- super().__init__()
27
-
28
- self.device = device
29
- self.filters = []
30
- self.num_views = 1
31
- filter_channels = cfg.MODEL.PyMAF.MLP_DIM
32
- self.last_op = nn.ReLU(True)
33
-
34
- for l in range(0, len(filter_channels) - 1):
35
- if 0 != l:
36
- self.filters.append(
37
- nn.Conv1d(filter_channels[l] + filter_channels[0],
38
- filter_channels[l + 1], 1))
39
- else:
40
- self.filters.append(
41
- nn.Conv1d(filter_channels[l], filter_channels[l + 1], 1))
42
-
43
- self.add_module("conv%d" % l, self.filters[l])
44
-
45
- self.im_feat = None
46
- self.cam = None
47
-
48
- # downsample SMPL mesh and assign part labels
49
- # from https://github.com/nkolot/GraphCMR/blob/master/data/mesh_downsampling.npz
50
- smpl_mesh_graph = np.load(MESH_DOWNSAMPLEING,
51
- allow_pickle=True,
52
- encoding='latin1')
53
-
54
- A = smpl_mesh_graph['A']
55
- U = smpl_mesh_graph['U']
56
- D = smpl_mesh_graph['D'] # shape: (2,)
57
-
58
- # downsampling
59
- ptD = []
60
- for i in range(len(D)):
61
- d = scipy.sparse.coo_matrix(D[i])
62
- i = torch.LongTensor(np.array([d.row, d.col]))
63
- v = torch.FloatTensor(d.data)
64
- ptD.append(torch.sparse.FloatTensor(i, v, d.shape))
65
-
66
- # downsampling mapping from 6890 points to 431 points
67
- # ptD[0].to_dense() - Size: [1723, 6890]
68
- # ptD[1].to_dense() - Size: [431. 1723]
69
- Dmap = torch.matmul(ptD[1].to_dense(),
70
- ptD[0].to_dense()) # 6890 -> 431
71
- self.register_buffer('Dmap', Dmap)
72
-
73
- def reduce_dim(self, feature):
74
- '''
75
- Dimension reduction by multi-layer perceptrons
76
- :param feature: list of [B, C_s, N] point-wise features before dimension reduction
77
- :return: [B, C_p x N] concatantion of point-wise features after dimension reduction
78
- '''
79
- y = feature
80
- tmpy = feature
81
- for i, f in enumerate(self.filters):
82
- y = self._modules['conv' +
83
- str(i)](y if i == 0 else torch.cat([y, tmpy], 1))
84
- if i != len(self.filters) - 1:
85
- y = F.leaky_relu(y)
86
- if self.num_views > 1 and i == len(self.filters) // 2:
87
- y = y.view(-1, self.num_views, y.shape[1],
88
- y.shape[2]).mean(dim=1)
89
- tmpy = feature.view(-1, self.num_views, feature.shape[1],
90
- feature.shape[2]).mean(dim=1)
91
-
92
- y = self.last_op(y)
93
-
94
- y = y.view(y.shape[0], -1)
95
- return y
96
-
97
- def sampling(self, points, im_feat=None, z_feat=None):
98
- '''
99
- Given 2D points, sample the point-wise features for each point,
100
- the dimension of point-wise features will be reduced from C_s to C_p by MLP.
101
- Image features should be pre-computed before this call.
102
- :param points: [B, N, 2] image coordinates of points
103
- :im_feat: [B, C_s, H_s, W_s] spatial feature maps
104
- :return: [B, C_p x N] concatantion of point-wise features after dimension reduction
105
- '''
106
- if im_feat is None:
107
- im_feat = self.im_feat
108
-
109
- batch_size = im_feat.shape[0]
110
-
111
- if version.parse(torch.__version__) >= version.parse('1.3.0'):
112
- # Default grid_sample behavior has changed to align_corners=False since 1.3.0.
113
- point_feat = torch.nn.functional.grid_sample(
114
- im_feat, points.unsqueeze(2), align_corners=True)[..., 0]
115
- else:
116
- point_feat = torch.nn.functional.grid_sample(
117
- im_feat, points.unsqueeze(2))[..., 0]
118
-
119
- mesh_align_feat = self.reduce_dim(point_feat)
120
- return mesh_align_feat
121
-
122
- def forward(self, p, s_feat=None, cam=None, **kwargs):
123
- ''' Returns mesh-aligned features for the 3D mesh points.
124
- Args:
125
- p (tensor): [B, N_m, 3] mesh vertices
126
- s_feat (tensor): [B, C_s, H_s, W_s] spatial feature maps
127
- cam (tensor): [B, 3] camera
128
- Return:
129
- mesh_align_feat (tensor): [B, C_p x N_m] mesh-aligned features
130
- '''
131
- if cam is None:
132
- cam = self.cam
133
- p_proj_2d = projection(p, cam, retain_z=False)
134
- mesh_align_feat = self.sampling(p_proj_2d, s_feat)
135
- return mesh_align_feat
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /models /res_module.py DELETED
@@ -1,385 +0,0 @@
1
- # code brought in part from https://github.com/microsoft/human-pose-estimation.pytorch/blob/master/lib/models/pose_resnet.py
2
-
3
- from __future__ import absolute_import
4
- from __future__ import division
5
- from __future__ import print_function
6
-
7
- import torch
8
- import torch.nn as nn
9
- import torch.nn.functional as F
10
- from collections import OrderedDict
11
- import os
12
- from lib.pymaf.core.cfgs import cfg
13
-
14
- import logging
15
-
16
- logger = logging.getLogger(__name__)
17
-
18
- BN_MOMENTUM = 0.1
19
-
20
-
21
- def conv3x3(in_planes, out_planes, stride=1, bias=False, groups=1):
22
- """3x3 convolution with padding"""
23
- return nn.Conv2d(in_planes * groups,
24
- out_planes * groups,
25
- kernel_size=3,
26
- stride=stride,
27
- padding=1,
28
- bias=bias,
29
- groups=groups)
30
-
31
-
32
- class BasicBlock(nn.Module):
33
- expansion = 1
34
-
35
- def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1):
36
- super().__init__()
37
- self.conv1 = conv3x3(inplanes, planes, stride, groups=groups)
38
- self.bn1 = nn.BatchNorm2d(planes * groups, momentum=BN_MOMENTUM)
39
- self.relu = nn.ReLU(inplace=True)
40
- self.conv2 = conv3x3(planes, planes, groups=groups)
41
- self.bn2 = nn.BatchNorm2d(planes * groups, momentum=BN_MOMENTUM)
42
- self.downsample = downsample
43
- self.stride = stride
44
-
45
- def forward(self, x):
46
- residual = x
47
-
48
- out = self.conv1(x)
49
- out = self.bn1(out)
50
- out = self.relu(out)
51
-
52
- out = self.conv2(out)
53
- out = self.bn2(out)
54
-
55
- if self.downsample is not None:
56
- residual = self.downsample(x)
57
-
58
- out += residual
59
- out = self.relu(out)
60
-
61
- return out
62
-
63
-
64
- class Bottleneck(nn.Module):
65
- expansion = 4
66
-
67
- def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1):
68
- super().__init__()
69
- self.conv1 = nn.Conv2d(inplanes * groups,
70
- planes * groups,
71
- kernel_size=1,
72
- bias=False,
73
- groups=groups)
74
- self.bn1 = nn.BatchNorm2d(planes * groups, momentum=BN_MOMENTUM)
75
- self.conv2 = nn.Conv2d(planes * groups,
76
- planes * groups,
77
- kernel_size=3,
78
- stride=stride,
79
- padding=1,
80
- bias=False,
81
- groups=groups)
82
- self.bn2 = nn.BatchNorm2d(planes * groups, momentum=BN_MOMENTUM)
83
- self.conv3 = nn.Conv2d(planes * groups,
84
- planes * self.expansion * groups,
85
- kernel_size=1,
86
- bias=False,
87
- groups=groups)
88
- self.bn3 = nn.BatchNorm2d(planes * self.expansion * groups,
89
- momentum=BN_MOMENTUM)
90
- self.relu = nn.ReLU(inplace=True)
91
- self.downsample = downsample
92
- self.stride = stride
93
-
94
- def forward(self, x):
95
- residual = x
96
-
97
- out = self.conv1(x)
98
- out = self.bn1(out)
99
- out = self.relu(out)
100
-
101
- out = self.conv2(out)
102
- out = self.bn2(out)
103
- out = self.relu(out)
104
-
105
- out = self.conv3(out)
106
- out = self.bn3(out)
107
-
108
- if self.downsample is not None:
109
- residual = self.downsample(x)
110
-
111
- out += residual
112
- out = self.relu(out)
113
-
114
- return out
115
-
116
-
117
- resnet_spec = {
118
- 18: (BasicBlock, [2, 2, 2, 2]),
119
- 34: (BasicBlock, [3, 4, 6, 3]),
120
- 50: (Bottleneck, [3, 4, 6, 3]),
121
- 101: (Bottleneck, [3, 4, 23, 3]),
122
- 152: (Bottleneck, [3, 8, 36, 3])
123
- }
124
-
125
-
126
- class IUV_predict_layer(nn.Module):
127
- def __init__(self,
128
- feat_dim=256,
129
- final_cov_k=3,
130
- part_out_dim=25,
131
- with_uv=True):
132
- super().__init__()
133
-
134
- self.with_uv = with_uv
135
- if self.with_uv:
136
- self.predict_u = nn.Conv2d(in_channels=feat_dim,
137
- out_channels=25,
138
- kernel_size=final_cov_k,
139
- stride=1,
140
- padding=1 if final_cov_k == 3 else 0)
141
-
142
- self.predict_v = nn.Conv2d(in_channels=feat_dim,
143
- out_channels=25,
144
- kernel_size=final_cov_k,
145
- stride=1,
146
- padding=1 if final_cov_k == 3 else 0)
147
-
148
- self.predict_ann_index = nn.Conv2d(
149
- in_channels=feat_dim,
150
- out_channels=15,
151
- kernel_size=final_cov_k,
152
- stride=1,
153
- padding=1 if final_cov_k == 3 else 0)
154
-
155
- self.predict_uv_index = nn.Conv2d(in_channels=feat_dim,
156
- out_channels=25,
157
- kernel_size=final_cov_k,
158
- stride=1,
159
- padding=1 if final_cov_k == 3 else 0)
160
-
161
- self.inplanes = feat_dim
162
-
163
- def _make_layer(self, block, planes, blocks, stride=1):
164
- downsample = None
165
- if stride != 1 or self.inplanes != planes * block.expansion:
166
- downsample = nn.Sequential(
167
- nn.Conv2d(self.inplanes,
168
- planes * block.expansion,
169
- kernel_size=1,
170
- stride=stride,
171
- bias=False),
172
- nn.BatchNorm2d(planes * block.expansion),
173
- )
174
-
175
- layers = []
176
- layers.append(block(self.inplanes, planes, stride, downsample))
177
- self.inplanes = planes * block.expansion
178
- for i in range(1, blocks):
179
- layers.append(block(self.inplanes, planes))
180
-
181
- return nn.Sequential(*layers)
182
-
183
- def forward(self, x):
184
- return_dict = {}
185
-
186
- predict_uv_index = self.predict_uv_index(x)
187
- predict_ann_index = self.predict_ann_index(x)
188
-
189
- return_dict['predict_uv_index'] = predict_uv_index
190
- return_dict['predict_ann_index'] = predict_ann_index
191
-
192
- if self.with_uv:
193
- predict_u = self.predict_u(x)
194
- predict_v = self.predict_v(x)
195
- return_dict['predict_u'] = predict_u
196
- return_dict['predict_v'] = predict_v
197
- else:
198
- return_dict['predict_u'] = None
199
- return_dict['predict_v'] = None
200
- # return_dict['predict_u'] = torch.zeros(predict_uv_index.shape).to(predict_uv_index.device)
201
- # return_dict['predict_v'] = torch.zeros(predict_uv_index.shape).to(predict_uv_index.device)
202
-
203
- return return_dict
204
-
205
-
206
- class SmplResNet(nn.Module):
207
- def __init__(self,
208
- resnet_nums,
209
- in_channels=3,
210
- num_classes=229,
211
- last_stride=2,
212
- n_extra_feat=0,
213
- truncate=0,
214
- **kwargs):
215
- super().__init__()
216
-
217
- self.inplanes = 64
218
- self.truncate = truncate
219
- # extra = cfg.MODEL.EXTRA
220
- # self.deconv_with_bias = extra.DECONV_WITH_BIAS
221
- block, layers = resnet_spec[resnet_nums]
222
-
223
- self.conv1 = nn.Conv2d(in_channels,
224
- 64,
225
- kernel_size=7,
226
- stride=2,
227
- padding=3,
228
- bias=False)
229
- self.bn1 = nn.BatchNorm2d(64, momentum=BN_MOMENTUM)
230
- self.relu = nn.ReLU(inplace=True)
231
- self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
232
- self.layer1 = self._make_layer(block, 64, layers[0])
233
- self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
234
- self.layer3 = self._make_layer(block, 256, layers[2],
235
- stride=2) if truncate < 2 else None
236
- self.layer4 = self._make_layer(
237
- block, 512, layers[3],
238
- stride=last_stride) if truncate < 1 else None
239
-
240
- self.avg_pooling = nn.AdaptiveAvgPool2d(1)
241
-
242
- self.num_classes = num_classes
243
- if num_classes > 0:
244
- self.final_layer = nn.Linear(512 * block.expansion, num_classes)
245
- nn.init.xavier_uniform_(self.final_layer.weight, gain=0.01)
246
-
247
- self.n_extra_feat = n_extra_feat
248
- if n_extra_feat > 0:
249
- self.trans_conv = nn.Sequential(
250
- nn.Conv2d(n_extra_feat + 512 * block.expansion,
251
- 512 * block.expansion,
252
- kernel_size=1,
253
- bias=False),
254
- nn.BatchNorm2d(512 * block.expansion, momentum=BN_MOMENTUM),
255
- nn.ReLU(True))
256
-
257
- def _make_layer(self, block, planes, blocks, stride=1):
258
- downsample = None
259
- if stride != 1 or self.inplanes != planes * block.expansion:
260
- downsample = nn.Sequential(
261
- nn.Conv2d(self.inplanes,
262
- planes * block.expansion,
263
- kernel_size=1,
264
- stride=stride,
265
- bias=False),
266
- nn.BatchNorm2d(planes * block.expansion, momentum=BN_MOMENTUM),
267
- )
268
-
269
- layers = []
270
- layers.append(block(self.inplanes, planes, stride, downsample))
271
- self.inplanes = planes * block.expansion
272
- for i in range(1, blocks):
273
- layers.append(block(self.inplanes, planes))
274
-
275
- return nn.Sequential(*layers)
276
-
277
- def forward(self, x, infeat=None):
278
- x = self.conv1(x)
279
- x = self.bn1(x)
280
- x = self.relu(x)
281
- x = self.maxpool(x)
282
-
283
- x1 = self.layer1(x)
284
- x2 = self.layer2(x1)
285
- x3 = self.layer3(x2) if self.truncate < 2 else x2
286
- x4 = self.layer4(x3) if self.truncate < 1 else x3
287
-
288
- if infeat is not None:
289
- x4 = self.trans_conv(torch.cat([infeat, x4], 1))
290
-
291
- if self.num_classes > 0:
292
- xp = self.avg_pooling(x4)
293
- cls = self.final_layer(xp.view(xp.size(0), -1))
294
- if not cfg.DANET.USE_MEAN_PARA:
295
- # for non-negative scale
296
- scale = F.relu(cls[:, 0]).unsqueeze(1)
297
- cls = torch.cat((scale, cls[:, 1:]), dim=1)
298
- else:
299
- cls = None
300
-
301
- return cls, {'x4': x4}
302
-
303
- def init_weights(self, pretrained=''):
304
- if os.path.isfile(pretrained):
305
- logger.info('=> loading pretrained model {}'.format(pretrained))
306
- # self.load_state_dict(pretrained_state_dict, strict=False)
307
- checkpoint = torch.load(pretrained)
308
- if isinstance(checkpoint, OrderedDict):
309
- # state_dict = checkpoint
310
- state_dict_old = self.state_dict()
311
- for key in state_dict_old.keys():
312
- if key in checkpoint.keys():
313
- if state_dict_old[key].shape != checkpoint[key].shape:
314
- del checkpoint[key]
315
- state_dict = checkpoint
316
- elif isinstance(checkpoint, dict) and 'state_dict' in checkpoint:
317
- state_dict_old = checkpoint['state_dict']
318
- state_dict = OrderedDict()
319
- # delete 'module.' because it is saved from DataParallel module
320
- for key in state_dict_old.keys():
321
- if key.startswith('module.'):
322
- # state_dict[key[7:]] = state_dict[key]
323
- # state_dict.pop(key)
324
- state_dict[key[7:]] = state_dict_old[key]
325
- else:
326
- state_dict[key] = state_dict_old[key]
327
- else:
328
- raise RuntimeError(
329
- 'No state_dict found in checkpoint file {}'.format(
330
- pretrained))
331
- self.load_state_dict(state_dict, strict=False)
332
- else:
333
- logger.error('=> imagenet pretrained model dose not exist')
334
- logger.error('=> please download it first')
335
- raise ValueError('imagenet pretrained model does not exist')
336
-
337
-
338
- class LimbResLayers(nn.Module):
339
- def __init__(self,
340
- resnet_nums,
341
- inplanes,
342
- outplanes=None,
343
- groups=1,
344
- **kwargs):
345
- super().__init__()
346
-
347
- self.inplanes = inplanes
348
- block, layers = resnet_spec[resnet_nums]
349
- self.outplanes = 512 if outplanes == None else outplanes
350
- self.layer4 = self._make_layer(block,
351
- self.outplanes,
352
- layers[3],
353
- stride=2,
354
- groups=groups)
355
-
356
- self.avg_pooling = nn.AdaptiveAvgPool2d(1)
357
-
358
- def _make_layer(self, block, planes, blocks, stride=1, groups=1):
359
- downsample = None
360
- if stride != 1 or self.inplanes != planes * block.expansion:
361
- downsample = nn.Sequential(
362
- nn.Conv2d(self.inplanes * groups,
363
- planes * block.expansion * groups,
364
- kernel_size=1,
365
- stride=stride,
366
- bias=False,
367
- groups=groups),
368
- nn.BatchNorm2d(planes * block.expansion * groups,
369
- momentum=BN_MOMENTUM),
370
- )
371
-
372
- layers = []
373
- layers.append(
374
- block(self.inplanes, planes, stride, downsample, groups=groups))
375
- self.inplanes = planes * block.expansion
376
- for i in range(1, blocks):
377
- layers.append(block(self.inplanes, planes, groups=groups))
378
-
379
- return nn.Sequential(*layers)
380
-
381
- def forward(self, x):
382
- x = self.layer4(x)
383
- x = self.avg_pooling(x)
384
-
385
- return x
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /utils / __init__.py DELETED
File without changes
lib / pymaf /utils / geometry.py DELETED
@@ -1,435 +0,0 @@
1
- import torch
2
- import numpy as np
3
- from torch.nn import functional as F
4
- """
5
- Useful geometric operations, e.g. Perspective projection and a differentiable Rodrigues formula
6
- Parts of the code are taken from https://github.com/MandyMo/pytorch_HMR
7
- """
8
-
9
-
10
- def batch_rodrigues(theta):
11
- """Convert axis-angle representation to rotation matrix.
12
- Args:
13
- theta: size = [B, 3]
14
- Returns:
15
- Rotation matrix corresponding to the quaternion -- size = [B, 3, 3]
16
- """
17
- l1norm = torch.norm(theta + 1e-8, p=2, dim=1)
18
- angle = torch.unsqueeze(l1norm, -1)
19
- normalized = torch.div(theta, angle)
20
- angle = angle * 0.5
21
- v_cos = torch.cos(angle)
22
- v_sin = torch.sin(angle)
23
- quat = torch.cat([v_cos, v_sin * normalized], dim=1)
24
- return quat_to_rotmat(quat)
25
-
26
-
27
- def quat_to_rotmat(quat):
28
- """Convert quaternion coefficients to rotation matrix.
29
- Args:
30
- quat: size = [B, 4] 4 <===>(w, x, y, z)
31
- Returns:
32
- Rotation matrix corresponding to the quaternion -- size = [B, 3, 3]
33
- """
34
- norm_quat = quat
35
- norm_quat = norm_quat / norm_quat.norm(p=2, dim=1, keepdim=True)
36
- w, x, y, z = norm_quat[:, 0], norm_quat[:, 1], norm_quat[:,
37
- 2], norm_quat[:,
38
- 3]
39
-
40
- B = quat.size(0)
41
-
42
- w2, x2, y2, z2 = w.pow(2), x.pow(2), y.pow(2), z.pow(2)
43
- wx, wy, wz = w * x, w * y, w * z
44
- xy, xz, yz = x * y, x * z, y * z
45
-
46
- rotMat = torch.stack([
47
- w2 + x2 - y2 - z2, 2 * xy - 2 * wz, 2 * wy + 2 * xz, 2 * wz + 2 * xy,
48
- w2 - x2 + y2 - z2, 2 * yz - 2 * wx, 2 * xz - 2 * wy, 2 * wx + 2 * yz,
49
- w2 - x2 - y2 + z2
50
- ],
51
- dim=1).view(B, 3, 3)
52
- return rotMat
53
-
54
-
55
- def rotation_matrix_to_angle_axis(rotation_matrix):
56
- """
57
- This function is borrowed from https://github.com/kornia/kornia
58
- Convert 3x4 rotation matrix to Rodrigues vector
59
- Args:
60
- rotation_matrix (Tensor): rotation matrix.
61
- Returns:
62
- Tensor: Rodrigues vector transformation.
63
- Shape:
64
- - Input: :math:`(N, 3, 4)`
65
- - Output: :math:`(N, 3)`
66
- Example:
67
- >>> input = torch.rand(2, 3, 4) # Nx4x4
68
- >>> output = tgm.rotation_matrix_to_angle_axis(input) # Nx3
69
- """
70
- if rotation_matrix.shape[1:] == (3, 3):
71
- rot_mat = rotation_matrix.reshape(-1, 3, 3)
72
- hom = torch.tensor([0, 0, 1],
73
- dtype=torch.float32,
74
- device=rotation_matrix.device).reshape(
75
- 1, 3, 1).expand(rot_mat.shape[0], -1, -1)
76
- rotation_matrix = torch.cat([rot_mat, hom], dim=-1)
77
-
78
- quaternion = rotation_matrix_to_quaternion(rotation_matrix)
79
- aa = quaternion_to_angle_axis(quaternion)
80
- aa[torch.isnan(aa)] = 0.0
81
- return aa
82
-
83
-
84
- def quaternion_to_angle_axis(quaternion: torch.Tensor) -> torch.Tensor:
85
- """
86
- This function is borrowed from https://github.com/kornia/kornia
87
- Convert quaternion vector to angle axis of rotation.
88
- Adapted from ceres C++ library: ceres-solver/include/ceres/rotation.h
89
- Args:
90
- quaternion (torch.Tensor): tensor with quaternions.
91
- Return:
92
- torch.Tensor: tensor with angle axis of rotation.
93
- Shape:
94
- - Input: :math:`(*, 4)` where `*` means, any number of dimensions
95
- - Output: :math:`(*, 3)`
96
- Example:
97
- >>> quaternion = torch.rand(2, 4) # Nx4
98
- >>> angle_axis = tgm.quaternion_to_angle_axis(quaternion) # Nx3
99
- """
100
- if not torch.is_tensor(quaternion):
101
- raise TypeError("Input type is not a torch.Tensor. Got {}".format(
102
- type(quaternion)))
103
-
104
- if not quaternion.shape[-1] == 4:
105
- raise ValueError(
106
- "Input must be a tensor of shape Nx4 or 4. Got {}".format(
107
- quaternion.shape))
108
- # unpack input and compute conversion
109
- q1: torch.Tensor = quaternion[..., 1]
110
- q2: torch.Tensor = quaternion[..., 2]
111
- q3: torch.Tensor = quaternion[..., 3]
112
- sin_squared_theta: torch.Tensor = q1 * q1 + q2 * q2 + q3 * q3
113
-
114
- sin_theta: torch.Tensor = torch.sqrt(sin_squared_theta)
115
- cos_theta: torch.Tensor = quaternion[..., 0]
116
- two_theta: torch.Tensor = 2.0 * torch.where(
117
- cos_theta < 0.0, torch.atan2(-sin_theta, -cos_theta),
118
- torch.atan2(sin_theta, cos_theta))
119
-
120
- k_pos: torch.Tensor = two_theta / sin_theta
121
- k_neg: torch.Tensor = 2.0 * torch.ones_like(sin_theta)
122
- k: torch.Tensor = torch.where(sin_squared_theta > 0.0, k_pos, k_neg)
123
-
124
- angle_axis: torch.Tensor = torch.zeros_like(quaternion)[..., :3]
125
- angle_axis[..., 0] += q1 * k
126
- angle_axis[..., 1] += q2 * k
127
- angle_axis[..., 2] += q3 * k
128
- return angle_axis
129
-
130
-
131
- def rotation_matrix_to_quaternion(rotation_matrix, eps=1e-6):
132
- """
133
- This function is borrowed from https://github.com/kornia/kornia
134
- Convert 3x4 rotation matrix to 4d quaternion vector
135
- This algorithm is based on algorithm described in
136
- https://github.com/KieranWynn/pyquaternion/blob/master/pyquaternion/quaternion.py#L201
137
- Args:
138
- rotation_matrix (Tensor): the rotation matrix to convert.
139
- Return:
140
- Tensor: the rotation in quaternion
141
- Shape:
142
- - Input: :math:`(N, 3, 4)`
143
- - Output: :math:`(N, 4)`
144
- Example:
145
- >>> input = torch.rand(4, 3, 4) # Nx3x4
146
- >>> output = tgm.rotation_matrix_to_quaternion(input) # Nx4
147
- """
148
- if not torch.is_tensor(rotation_matrix):
149
- raise TypeError("Input type is not a torch.Tensor. Got {}".format(
150
- type(rotation_matrix)))
151
-
152
- if len(rotation_matrix.shape) > 3:
153
- raise ValueError(
154
- "Input size must be a three dimensional tensor. Got {}".format(
155
- rotation_matrix.shape))
156
- if not rotation_matrix.shape[-2:] == (3, 4):
157
- raise ValueError(
158
- "Input size must be a N x 3 x 4 tensor. Got {}".format(
159
- rotation_matrix.shape))
160
-
161
- rmat_t = torch.transpose(rotation_matrix, 1, 2)
162
-
163
- mask_d2 = rmat_t[:, 2, 2] < eps
164
-
165
- mask_d0_d1 = rmat_t[:, 0, 0] > rmat_t[:, 1, 1]
166
- mask_d0_nd1 = rmat_t[:, 0, 0] < -rmat_t[:, 1, 1]
167
-
168
- t0 = 1 + rmat_t[:, 0, 0] - rmat_t[:, 1, 1] - rmat_t[:, 2, 2]
169
- q0 = torch.stack([
170
- rmat_t[:, 1, 2] - rmat_t[:, 2, 1], t0,
171
- rmat_t[:, 0, 1] + rmat_t[:, 1, 0], rmat_t[:, 2, 0] + rmat_t[:, 0, 2]
172
- ], -1)
173
- t0_rep = t0.repeat(4, 1).t()
174
-
175
- t1 = 1 - rmat_t[:, 0, 0] + rmat_t[:, 1, 1] - rmat_t[:, 2, 2]
176
- q1 = torch.stack([
177
- rmat_t[:, 2, 0] - rmat_t[:, 0, 2], rmat_t[:, 0, 1] + rmat_t[:, 1, 0],
178
- t1, rmat_t[:, 1, 2] + rmat_t[:, 2, 1]
179
- ], -1)
180
- t1_rep = t1.repeat(4, 1).t()
181
-
182
- t2 = 1 - rmat_t[:, 0, 0] - rmat_t[:, 1, 1] + rmat_t[:, 2, 2]
183
- q2 = torch.stack([
184
- rmat_t[:, 0, 1] - rmat_t[:, 1, 0], rmat_t[:, 2, 0] + rmat_t[:, 0, 2],
185
- rmat_t[:, 1, 2] + rmat_t[:, 2, 1], t2
186
- ], -1)
187
- t2_rep = t2.repeat(4, 1).t()
188
-
189
- t3 = 1 + rmat_t[:, 0, 0] + rmat_t[:, 1, 1] + rmat_t[:, 2, 2]
190
- q3 = torch.stack([
191
- t3, rmat_t[:, 1, 2] - rmat_t[:, 2, 1],
192
- rmat_t[:, 2, 0] - rmat_t[:, 0, 2], rmat_t[:, 0, 1] - rmat_t[:, 1, 0]
193
- ], -1)
194
- t3_rep = t3.repeat(4, 1).t()
195
-
196
- mask_c0 = mask_d2 * mask_d0_d1
197
- mask_c1 = mask_d2 * ~mask_d0_d1
198
- mask_c2 = ~mask_d2 * mask_d0_nd1
199
- mask_c3 = ~mask_d2 * ~mask_d0_nd1
200
- mask_c0 = mask_c0.view(-1, 1).type_as(q0)
201
- mask_c1 = mask_c1.view(-1, 1).type_as(q1)
202
- mask_c2 = mask_c2.view(-1, 1).type_as(q2)
203
- mask_c3 = mask_c3.view(-1, 1).type_as(q3)
204
-
205
- q = q0 * mask_c0 + q1 * mask_c1 + q2 * mask_c2 + q3 * mask_c3
206
- q /= torch.sqrt(t0_rep * mask_c0 + t1_rep * mask_c1 + # noqa
207
- t2_rep * mask_c2 + t3_rep * mask_c3) # noqa
208
- q *= 0.5
209
- return q
210
-
211
-
212
- def rot6d_to_rotmat(x):
213
- """Convert 6D rotation representation to 3x3 rotation matrix.
214
- Based on Zhou et al., "On the Continuity of Rotation Representations in Neural Networks", CVPR 2019
215
- Input:
216
- (B,6) Batch of 6-D rotation representations
217
- Output:
218
- (B,3,3) Batch of corresponding rotation matrices
219
- """
220
- x = x.view(-1, 3, 2)
221
- a1 = x[:, :, 0]
222
- a2 = x[:, :, 1]
223
- b1 = F.normalize(a1)
224
- b2 = F.normalize(a2 - torch.einsum('bi,bi->b', b1, a2).unsqueeze(-1) * b1)
225
- b3 = torch.cross(b1, b2)
226
- return torch.stack((b1, b2, b3), dim=-1)
227
-
228
-
229
- def projection(pred_joints, pred_camera, retain_z=False):
230
- pred_cam_t = torch.stack([
231
- pred_camera[:, 1], pred_camera[:, 2], 2 * 5000. /
232
- (224. * pred_camera[:, 0] + 1e-9)
233
- ],
234
- dim=-1)
235
- batch_size = pred_joints.shape[0]
236
- camera_center = torch.zeros(batch_size, 2)
237
- pred_keypoints_2d = perspective_projection(
238
- pred_joints,
239
- rotation=torch.eye(3).unsqueeze(0).expand(batch_size, -1,
240
- -1).to(pred_joints.device),
241
- translation=pred_cam_t,
242
- focal_length=5000.,
243
- camera_center=camera_center,
244
- retain_z=retain_z)
245
- # Normalize keypoints to [-1,1]
246
- pred_keypoints_2d = pred_keypoints_2d / (224. / 2.)
247
- return pred_keypoints_2d
248
-
249
-
250
- def perspective_projection(points,
251
- rotation,
252
- translation,
253
- focal_length,
254
- camera_center,
255
- retain_z=False):
256
- """
257
- This function computes the perspective projection of a set of points.
258
- Input:
259
- points (bs, N, 3): 3D points
260
- rotation (bs, 3, 3): Camera rotation
261
- translation (bs, 3): Camera translation
262
- focal_length (bs,) or scalar: Focal length
263
- camera_center (bs, 2): Camera center
264
- """
265
- batch_size = points.shape[0]
266
- K = torch.zeros([batch_size, 3, 3], device=points.device)
267
- K[:, 0, 0] = focal_length
268
- K[:, 1, 1] = focal_length
269
- K[:, 2, 2] = 1.
270
- K[:, :-1, -1] = camera_center
271
-
272
- # Transform points
273
- points = torch.einsum('bij,bkj->bki', rotation, points)
274
- points = points + translation.unsqueeze(1)
275
-
276
- # Apply perspective distortion
277
- projected_points = points / points[:, :, -1].unsqueeze(-1)
278
-
279
- # Apply camera intrinsics
280
- projected_points = torch.einsum('bij,bkj->bki', K, projected_points)
281
-
282
- if retain_z:
283
- return projected_points
284
- else:
285
- return projected_points[:, :, :-1]
286
-
287
-
288
- def estimate_translation_np(S,
289
- joints_2d,
290
- joints_conf,
291
- focal_length=5000,
292
- img_size=224):
293
- """Find camera translation that brings 3D joints S closest to 2D the corresponding joints_2d.
294
- Input:
295
- S: (25, 3) 3D joint locations
296
- joints: (25, 3) 2D joint locations and confidence
297
- Returns:
298
- (3,) camera translation vector
299
- """
300
-
301
- num_joints = S.shape[0]
302
- # focal length
303
- f = np.array([focal_length, focal_length])
304
- # optical center
305
- center = np.array([img_size / 2., img_size / 2.])
306
-
307
- # transformations
308
- Z = np.reshape(np.tile(S[:, 2], (2, 1)).T, -1)
309
- XY = np.reshape(S[:, 0:2], -1)
310
- O = np.tile(center, num_joints)
311
- F = np.tile(f, num_joints)
312
- weight2 = np.reshape(np.tile(np.sqrt(joints_conf), (2, 1)).T, -1)
313
-
314
- # least squares
315
- Q = np.array([
316
- F * np.tile(np.array([1, 0]), num_joints),
317
- F * np.tile(np.array([0, 1]), num_joints),
318
- O - np.reshape(joints_2d, -1)
319
- ]).T
320
- c = (np.reshape(joints_2d, -1) - O) * Z - F * XY
321
-
322
- # weighted least squares
323
- W = np.diagflat(weight2)
324
- Q = np.dot(W, Q)
325
- c = np.dot(W, c)
326
-
327
- # square matrix
328
- A = np.dot(Q.T, Q)
329
- b = np.dot(Q.T, c)
330
-
331
- # solution
332
- trans = np.linalg.solve(A, b)
333
-
334
- return trans
335
-
336
-
337
- def estimate_translation(S, joints_2d, focal_length=5000., img_size=224.):
338
- """Find camera translation that brings 3D joints S closest to 2D the corresponding joints_2d.
339
- Input:
340
- S: (B, 49, 3) 3D joint locations
341
- joints: (B, 49, 3) 2D joint locations and confidence
342
- Returns:
343
- (B, 3) camera translation vectors
344
- """
345
-
346
- device = S.device
347
- # Use only joints 25:49 (GT joints)
348
- S = S[:, 25:, :].cpu().numpy()
349
- joints_2d = joints_2d[:, 25:, :].cpu().numpy()
350
- joints_conf = joints_2d[:, :, -1]
351
- joints_2d = joints_2d[:, :, :-1]
352
- trans = np.zeros((S.shape[0], 3), dtype=np.float32)
353
- # Find the translation for each example in the batch
354
- for i in range(S.shape[0]):
355
- S_i = S[i]
356
- joints_i = joints_2d[i]
357
- conf_i = joints_conf[i]
358
- trans[i] = estimate_translation_np(S_i,
359
- joints_i,
360
- conf_i,
361
- focal_length=focal_length,
362
- img_size=img_size)
363
- return torch.from_numpy(trans).to(device)
364
-
365
-
366
- def Rot_y(angle, category='torch', prepend_dim=True, device=None):
367
- '''Rotate around y-axis by angle
368
- Args:
369
- category: 'torch' or 'numpy'
370
- prepend_dim: prepend an extra dimension
371
- Return: Rotation matrix with shape [1, 3, 3] (prepend_dim=True)
372
- '''
373
- m = np.array([[np.cos(angle), 0., np.sin(angle)], [0., 1., 0.],
374
- [-np.sin(angle), 0., np.cos(angle)]])
375
- if category == 'torch':
376
- if prepend_dim:
377
- return torch.tensor(m, dtype=torch.float,
378
- device=device).unsqueeze(0)
379
- else:
380
- return torch.tensor(m, dtype=torch.float, device=device)
381
- elif category == 'numpy':
382
- if prepend_dim:
383
- return np.expand_dims(m, 0)
384
- else:
385
- return m
386
- else:
387
- raise ValueError("category must be 'torch' or 'numpy'")
388
-
389
-
390
- def Rot_x(angle, category='torch', prepend_dim=True, device=None):
391
- '''Rotate around x-axis by angle
392
- Args:
393
- category: 'torch' or 'numpy'
394
- prepend_dim: prepend an extra dimension
395
- Return: Rotation matrix with shape [1, 3, 3] (prepend_dim=True)
396
- '''
397
- m = np.array([[1., 0., 0.], [0., np.cos(angle), -np.sin(angle)],
398
- [0., np.sin(angle), np.cos(angle)]])
399
- if category == 'torch':
400
- if prepend_dim:
401
- return torch.tensor(m, dtype=torch.float,
402
- device=device).unsqueeze(0)
403
- else:
404
- return torch.tensor(m, dtype=torch.float, device=device)
405
- elif category == 'numpy':
406
- if prepend_dim:
407
- return np.expand_dims(m, 0)
408
- else:
409
- return m
410
- else:
411
- raise ValueError("category must be 'torch' or 'numpy'")
412
-
413
-
414
- def Rot_z(angle, category='torch', prepend_dim=True, device=None):
415
- '''Rotate around z-axis by angle
416
- Args:
417
- category: 'torch' or 'numpy'
418
- prepend_dim: prepend an extra dimension
419
- Return: Rotation matrix with shape [1, 3, 3] (prepend_dim=True)
420
- '''
421
- m = np.array([[np.cos(angle), -np.sin(angle), 0.],
422
- [np.sin(angle), np.cos(angle), 0.], [0., 0., 1.]])
423
- if category == 'torch':
424
- if prepend_dim:
425
- return torch.tensor(m, dtype=torch.float,
426
- device=device).unsqueeze(0)
427
- else:
428
- return torch.tensor(m, dtype=torch.float, device=device)
429
- elif category == 'numpy':
430
- if prepend_dim:
431
- return np.expand_dims(m, 0)
432
- else:
433
- return m
434
- else:
435
- raise ValueError("category must be 'torch' or 'numpy'")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /utils / imutils.py DELETED
@@ -1,491 +0,0 @@
1
- """
2
- This file contains functions that are used to perform data augmentation.
3
- """
4
- import cv2
5
- import io
6
- import torch
7
- import numpy as np
8
- from PIL import Image
9
- from rembg import remove
10
- from rembg.session_factory import new_session
11
- from torchvision.models import detection
12
-
13
- from lib.pymaf.core import constants
14
- from lib.pymaf.utils.streamer import aug_matrix
15
- from lib.common.cloth_extraction import load_segmentation
16
- from torchvision import transforms
17
-
18
-
19
- def load_img(img_file):
20
-
21
- img = cv2.imread(img_file, cv2.IMREAD_UNCHANGED)
22
- if len(img.shape) == 2:
23
- img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
24
-
25
- if not img_file.endswith("png"):
26
- img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
27
- else:
28
- img = cv2.cvtColor(img, cv2.COLOR_RGBA2BGR)
29
-
30
- return img
31
-
32
-
33
- def get_bbox(img, det):
34
-
35
- input = np.float32(img)
36
- input = (input / 255.0 -
37
- (0.5, 0.5, 0.5)) / (0.5, 0.5, 0.5) # TO [-1.0, 1.0]
38
- input = input.transpose(2, 0, 1) # TO [3 x H x W]
39
- bboxes, probs = det(torch.from_numpy(input).float().unsqueeze(0))
40
-
41
- probs = probs.unsqueeze(3)
42
- bboxes = (bboxes * probs).sum(dim=1, keepdim=True) / probs.sum(
43
- dim=1, keepdim=True)
44
- bbox = bboxes[0, 0, 0].cpu().numpy()
45
-
46
- return bbox
47
- # Michael Black is
48
-
49
-
50
- def get_transformer(input_res):
51
-
52
- image_to_tensor = transforms.Compose([
53
- transforms.Resize(input_res),
54
- transforms.ToTensor(),
55
- transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
56
- ])
57
-
58
- mask_to_tensor = transforms.Compose([
59
- transforms.Resize(input_res),
60
- transforms.ToTensor(),
61
- transforms.Normalize((0.0, ), (1.0, ))
62
- ])
63
-
64
- image_to_pymaf_tensor = transforms.Compose([
65
- transforms.Resize(size=224),
66
- transforms.Normalize(mean=constants.IMG_NORM_MEAN,
67
- std=constants.IMG_NORM_STD)
68
- ])
69
-
70
- image_to_pixie_tensor = transforms.Compose([
71
- transforms.Resize(224)
72
- ])
73
-
74
- def image_to_hybrik_tensor(img):
75
- # mean
76
- img[0].add_(-0.406)
77
- img[1].add_(-0.457)
78
- img[2].add_(-0.480)
79
-
80
- # std
81
- img[0].div_(0.225)
82
- img[1].div_(0.224)
83
- img[2].div_(0.229)
84
- return img
85
-
86
- return [image_to_tensor, mask_to_tensor, image_to_pymaf_tensor, image_to_pixie_tensor, image_to_hybrik_tensor]
87
-
88
-
89
- def process_image(img_file, hps_type, input_res=512, device=None, seg_path=None):
90
- """Read image, do preprocessing and possibly crop it according to the bounding box.
91
- If there are bounding box annotations, use them to crop the image.
92
- If no bounding box is specified but openpose detections are available, use them to get the bounding box.
93
- """
94
-
95
- [image_to_tensor, mask_to_tensor, image_to_pymaf_tensor,
96
- image_to_pixie_tensor, image_to_hybrik_tensor] = get_transformer(input_res)
97
-
98
- img_ori = load_img(img_file)
99
-
100
- in_height, in_width, _ = img_ori.shape
101
- M = aug_matrix(in_width, in_height, input_res*2, input_res*2)
102
-
103
- # from rectangle to square
104
- img_for_crop = cv2.warpAffine(img_ori, M[0:2, :],
105
- (input_res*2, input_res*2), flags=cv2.INTER_CUBIC)
106
-
107
- # detection for bbox
108
- detector = detection.maskrcnn_resnet50_fpn(pretrained=True)
109
- detector.eval()
110
- predictions = detector(
111
- [torch.from_numpy(img_for_crop).permute(2, 0, 1) / 255.])[0]
112
- human_ids = torch.where(
113
- predictions["scores"] == predictions["scores"][predictions['labels'] == 1].max())
114
- bbox = predictions["boxes"][human_ids, :].flatten().detach().cpu().numpy()
115
-
116
- width = bbox[2] - bbox[0]
117
- height = bbox[3] - bbox[1]
118
- center = np.array([(bbox[0] + bbox[2]) / 2.0,
119
- (bbox[1] + bbox[3]) / 2.0])
120
-
121
- scale = max(height, width) / 180
122
-
123
- if hps_type == 'hybrik':
124
- img_np = crop_for_hybrik(img_for_crop, center,
125
- np.array([scale * 180, scale * 180]))
126
- else:
127
- img_np, cropping_parameters = crop(
128
- img_for_crop, center, scale, (input_res, input_res))
129
-
130
- img_pil = Image.fromarray(remove(img_np, post_process_mask=True, session=new_session("u2net")))
131
-
132
- # for icon
133
- img_rgb = image_to_tensor(img_pil.convert("RGB"))
134
- img_mask = torch.tensor(1.0) - (mask_to_tensor(img_pil.split()[-1]) <
135
- torch.tensor(0.5)).float()
136
- img_tensor = img_rgb * img_mask
137
-
138
- # for hps
139
- img_hps = img_np.astype(np.float32) / 255.
140
- img_hps = torch.from_numpy(img_hps).permute(2, 0, 1)
141
-
142
- if hps_type == 'bev':
143
- img_hps = img_np[:, :, [2, 1, 0]]
144
- elif hps_type == 'hybrik':
145
- img_hps = image_to_hybrik_tensor(img_hps).unsqueeze(0).to(device)
146
- elif hps_type != 'pixie':
147
- img_hps = image_to_pymaf_tensor(img_hps).unsqueeze(0).to(device)
148
- else:
149
- img_hps = image_to_pixie_tensor(img_hps).unsqueeze(0).to(device)
150
-
151
- # uncrop params
152
- uncrop_param = {'center': center,
153
- 'scale': scale,
154
- 'ori_shape': img_ori.shape,
155
- 'box_shape': img_np.shape,
156
- 'crop_shape': img_for_crop.shape,
157
- 'M': M}
158
-
159
- if not (seg_path is None):
160
- segmentations = load_segmentation(seg_path, (in_height, in_width))
161
- seg_coord_normalized = []
162
- for seg in segmentations:
163
- coord_normalized = []
164
- for xy in seg['coordinates']:
165
- xy_h = np.vstack((xy[:, 0], xy[:, 1], np.ones(len(xy)))).T
166
- warped_indeces = M[0:2, :] @ xy_h[:, :, None]
167
- warped_indeces = np.array(warped_indeces).astype(int)
168
- warped_indeces.resize((warped_indeces.shape[:2]))
169
-
170
- # cropped_indeces = crop_segmentation(warped_indeces, center, scale, (input_res, input_res), img_np.shape)
171
- cropped_indeces = crop_segmentation(
172
- warped_indeces, (input_res, input_res), cropping_parameters)
173
-
174
- indices = np.vstack(
175
- (cropped_indeces[:, 0], cropped_indeces[:, 1])).T
176
-
177
- # Convert to NDC coordinates
178
- seg_cropped_normalized = 2*(indices / input_res) - 1
179
- # Don't know why we need to divide by 50 but it works ¯\_(ツ)_/¯ (probably some scaling factor somewhere)
180
- # Divide only by 45 on the horizontal axis to take the curve of the human body into account
181
- seg_cropped_normalized[:, 0] = (
182
- 1/40) * seg_cropped_normalized[:, 0]
183
- seg_cropped_normalized[:, 1] = (
184
- 1/50) * seg_cropped_normalized[:, 1]
185
- coord_normalized.append(seg_cropped_normalized)
186
-
187
- seg['coord_normalized'] = coord_normalized
188
- seg_coord_normalized.append(seg)
189
-
190
- return img_tensor, img_hps, img_ori, img_mask, uncrop_param, seg_coord_normalized
191
-
192
- return img_tensor, img_hps, img_ori, img_mask, uncrop_param
193
-
194
-
195
- def get_transform(center, scale, res):
196
- """Generate transformation matrix."""
197
- h = 200 * scale
198
- t = np.zeros((3, 3))
199
- t[0, 0] = float(res[1]) / h
200
- t[1, 1] = float(res[0]) / h
201
- t[0, 2] = res[1] * (-float(center[0]) / h + .5)
202
- t[1, 2] = res[0] * (-float(center[1]) / h + .5)
203
- t[2, 2] = 1
204
-
205
- return t
206
-
207
-
208
- def transform(pt, center, scale, res, invert=0):
209
- """Transform pixel location to different reference."""
210
- t = get_transform(center, scale, res)
211
- if invert:
212
- t = np.linalg.inv(t)
213
- new_pt = np.array([pt[0] - 1, pt[1] - 1, 1.]).T
214
- new_pt = np.dot(t, new_pt)
215
- return np.around(new_pt[:2]).astype(np.int16)
216
-
217
-
218
- def crop(img, center, scale, res):
219
- """Crop image according to the supplied bounding box."""
220
-
221
- # Upper left point
222
- ul = np.array(transform([0, 0], center, scale, res, invert=1))
223
-
224
- # Bottom right point
225
- br = np.array(transform(res, center, scale, res, invert=1))
226
-
227
- new_shape = [br[1] - ul[1], br[0] - ul[0]]
228
- if len(img.shape) > 2:
229
- new_shape += [img.shape[2]]
230
- new_img = np.zeros(new_shape)
231
-
232
- # Range to fill new array
233
- new_x = max(0, -ul[0]), min(br[0], len(img[0])) - ul[0]
234
- new_y = max(0, -ul[1]), min(br[1], len(img)) - ul[1]
235
-
236
- # Range to sample from original image
237
- old_x = max(0, ul[0]), min(len(img[0]), br[0])
238
- old_y = max(0, ul[1]), min(len(img), br[1])
239
-
240
- new_img[new_y[0]:new_y[1], new_x[0]:new_x[1]
241
- ] = img[old_y[0]:old_y[1], old_x[0]:old_x[1]]
242
- if len(img.shape) == 2:
243
- new_img = np.array(Image.fromarray(new_img).resize(res))
244
- else:
245
- new_img = np.array(Image.fromarray(
246
- new_img.astype(np.uint8)).resize(res))
247
-
248
- return new_img, (old_x, new_x, old_y, new_y, new_shape)
249
-
250
-
251
- def crop_segmentation(org_coord, res, cropping_parameters):
252
- old_x, new_x, old_y, new_y, new_shape = cropping_parameters
253
-
254
- new_coord = np.zeros((org_coord.shape))
255
- new_coord[:, 0] = new_x[0] + (org_coord[:, 0] - old_x[0])
256
- new_coord[:, 1] = new_y[0] + (org_coord[:, 1] - old_y[0])
257
-
258
- new_coord[:, 0] = res[0] * (new_coord[:, 0] / new_shape[1])
259
- new_coord[:, 1] = res[1] * (new_coord[:, 1] / new_shape[0])
260
-
261
- return new_coord
262
-
263
-
264
- def crop_for_hybrik(img, center, scale):
265
- inp_h, inp_w = (256, 256)
266
- trans = get_affine_transform(center, scale, 0, [inp_w, inp_h])
267
- new_img = cv2.warpAffine(
268
- img, trans, (int(inp_w), int(inp_h)), flags=cv2.INTER_LINEAR)
269
- return new_img
270
-
271
-
272
- def get_affine_transform(center,
273
- scale,
274
- rot,
275
- output_size,
276
- shift=np.array([0, 0], dtype=np.float32),
277
- inv=0):
278
-
279
- def get_dir(src_point, rot_rad):
280
- """Rotate the point by `rot_rad` degree."""
281
- sn, cs = np.sin(rot_rad), np.cos(rot_rad)
282
-
283
- src_result = [0, 0]
284
- src_result[0] = src_point[0] * cs - src_point[1] * sn
285
- src_result[1] = src_point[0] * sn + src_point[1] * cs
286
-
287
- return src_result
288
-
289
- def get_3rd_point(a, b):
290
- """Return vector c that perpendicular to (a - b)."""
291
- direct = a - b
292
- return b + np.array([-direct[1], direct[0]], dtype=np.float32)
293
-
294
- if not isinstance(scale, np.ndarray) and not isinstance(scale, list):
295
- scale = np.array([scale, scale])
296
-
297
- scale_tmp = scale
298
- src_w = scale_tmp[0]
299
- dst_w = output_size[0]
300
- dst_h = output_size[1]
301
-
302
- rot_rad = np.pi * rot / 180
303
- src_dir = get_dir([0, src_w * -0.5], rot_rad)
304
- dst_dir = np.array([0, dst_w * -0.5], np.float32)
305
-
306
- src = np.zeros((3, 2), dtype=np.float32)
307
- dst = np.zeros((3, 2), dtype=np.float32)
308
- src[0, :] = center + scale_tmp * shift
309
- src[1, :] = center + src_dir + scale_tmp * shift
310
- dst[0, :] = [dst_w * 0.5, dst_h * 0.5]
311
- dst[1, :] = np.array([dst_w * 0.5, dst_h * 0.5]) + dst_dir
312
-
313
- src[2:, :] = get_3rd_point(src[0, :], src[1, :])
314
- dst[2:, :] = get_3rd_point(dst[0, :], dst[1, :])
315
-
316
- if inv:
317
- trans = cv2.getAffineTransform(np.float32(dst), np.float32(src))
318
- else:
319
- trans = cv2.getAffineTransform(np.float32(src), np.float32(dst))
320
-
321
- return trans
322
-
323
-
324
- def corner_align(ul, br):
325
-
326
- if ul[1]-ul[0] != br[1]-br[0]:
327
- ul[1] = ul[0]+br[1]-br[0]
328
-
329
- return ul, br
330
-
331
-
332
- def uncrop(img, center, scale, orig_shape):
333
- """'Undo' the image cropping/resizing.
334
- This function is used when evaluating mask/part segmentation.
335
- """
336
-
337
- res = img.shape[:2]
338
-
339
- # Upper left point
340
- ul = np.array(transform([0, 0], center, scale, res, invert=1))
341
- # Bottom right point
342
- br = np.array(transform(res, center, scale, res, invert=1))
343
-
344
- # quick fix
345
- ul, br = corner_align(ul, br)
346
-
347
- # size of cropped image
348
- crop_shape = [br[1] - ul[1], br[0] - ul[0]]
349
- new_img = np.zeros(orig_shape, dtype=np.uint8)
350
-
351
- # Range to fill new array
352
- new_x = max(0, -ul[0]), min(br[0], orig_shape[1]) - ul[0]
353
- new_y = max(0, -ul[1]), min(br[1], orig_shape[0]) - ul[1]
354
-
355
- # Range to sample from original image
356
- old_x = max(0, ul[0]), min(orig_shape[1], br[0])
357
- old_y = max(0, ul[1]), min(orig_shape[0], br[1])
358
-
359
- img = np.array(Image.fromarray(img.astype(np.uint8)).resize(crop_shape))
360
-
361
- new_img[old_y[0]:old_y[1], old_x[0]:old_x[1]
362
- ] = img[new_y[0]:new_y[1], new_x[0]:new_x[1]]
363
-
364
- return new_img
365
-
366
-
367
- def rot_aa(aa, rot):
368
- """Rotate axis angle parameters."""
369
- # pose parameters
370
- R = np.array([[np.cos(np.deg2rad(-rot)), -np.sin(np.deg2rad(-rot)), 0],
371
- [np.sin(np.deg2rad(-rot)),
372
- np.cos(np.deg2rad(-rot)), 0], [0, 0, 1]])
373
- # find the rotation of the body in camera frame
374
- per_rdg, _ = cv2.Rodrigues(aa)
375
- # apply the global rotation to the global orientation
376
- resrot, _ = cv2.Rodrigues(np.dot(R, per_rdg))
377
- aa = (resrot.T)[0]
378
- return aa
379
-
380
-
381
- def flip_img(img):
382
- """Flip rgb images or masks.
383
- channels come last, e.g. (256,256,3).
384
- """
385
- img = np.fliplr(img)
386
- return img
387
-
388
-
389
- def flip_kp(kp, is_smpl=False):
390
- """Flip keypoints."""
391
- if len(kp) == 24:
392
- if is_smpl:
393
- flipped_parts = constants.SMPL_JOINTS_FLIP_PERM
394
- else:
395
- flipped_parts = constants.J24_FLIP_PERM
396
- elif len(kp) == 49:
397
- if is_smpl:
398
- flipped_parts = constants.SMPL_J49_FLIP_PERM
399
- else:
400
- flipped_parts = constants.J49_FLIP_PERM
401
- kp = kp[flipped_parts]
402
- kp[:, 0] = -kp[:, 0]
403
- return kp
404
-
405
-
406
- def flip_pose(pose):
407
- """Flip pose.
408
- The flipping is based on SMPL parameters.
409
- """
410
- flipped_parts = constants.SMPL_POSE_FLIP_PERM
411
- pose = pose[flipped_parts]
412
- # we also negate the second and the third dimension of the axis-angle
413
- pose[1::3] = -pose[1::3]
414
- pose[2::3] = -pose[2::3]
415
- return pose
416
-
417
-
418
- def normalize_2d_kp(kp_2d, crop_size=224, inv=False):
419
- # Normalize keypoints between -1, 1
420
- if not inv:
421
- ratio = 1.0 / crop_size
422
- kp_2d = 2.0 * kp_2d * ratio - 1.0
423
- else:
424
- ratio = 1.0 / crop_size
425
- kp_2d = (kp_2d + 1.0) / (2 * ratio)
426
-
427
- return kp_2d
428
-
429
-
430
- def generate_heatmap(joints, heatmap_size, sigma=1, joints_vis=None):
431
- '''
432
- param joints: [num_joints, 3]
433
- param joints_vis: [num_joints, 3]
434
- return: target, target_weight(1: visible, 0: invisible)
435
- '''
436
- num_joints = joints.shape[0]
437
- device = joints.device
438
- cur_device = torch.device(device.type, device.index)
439
- if not hasattr(heatmap_size, '__len__'):
440
- # width height
441
- heatmap_size = [heatmap_size, heatmap_size]
442
- assert len(heatmap_size) == 2
443
- target_weight = np.ones((num_joints, 1), dtype=np.float32)
444
- if joints_vis is not None:
445
- target_weight[:, 0] = joints_vis[:, 0]
446
- target = torch.zeros((num_joints, heatmap_size[1], heatmap_size[0]),
447
- dtype=torch.float32,
448
- device=cur_device)
449
-
450
- tmp_size = sigma * 3
451
-
452
- for joint_id in range(num_joints):
453
- mu_x = int(joints[joint_id][0] * heatmap_size[0] + 0.5)
454
- mu_y = int(joints[joint_id][1] * heatmap_size[1] + 0.5)
455
- # Check that any part of the gaussian is in-bounds
456
- ul = [int(mu_x - tmp_size), int(mu_y - tmp_size)]
457
- br = [int(mu_x + tmp_size + 1), int(mu_y + tmp_size + 1)]
458
- if ul[0] >= heatmap_size[0] or ul[1] >= heatmap_size[1] \
459
- or br[0] < 0 or br[1] < 0:
460
- # If not, just return the image as is
461
- target_weight[joint_id] = 0
462
- continue
463
-
464
- # # Generate gaussian
465
- size = 2 * tmp_size + 1
466
- # x = np.arange(0, size, 1, np.float32)
467
- # y = x[:, np.newaxis]
468
- # x0 = y0 = size // 2
469
- # # The gaussian is not normalized, we want the center value to equal 1
470
- # g = np.exp(- ((x - x0) ** 2 + (y - y0) ** 2) / (2 * sigma ** 2))
471
- # g = torch.from_numpy(g.astype(np.float32))
472
-
473
- x = torch.arange(0, size, dtype=torch.float32, device=cur_device)
474
- y = x.unsqueeze(-1)
475
- x0 = y0 = size // 2
476
- # The gaussian is not normalized, we want the center value to equal 1
477
- g = torch.exp(-((x - x0)**2 + (y - y0)**2) / (2 * sigma**2))
478
-
479
- # Usable gaussian range
480
- g_x = max(0, -ul[0]), min(br[0], heatmap_size[0]) - ul[0]
481
- g_y = max(0, -ul[1]), min(br[1], heatmap_size[1]) - ul[1]
482
- # Image range
483
- img_x = max(0, ul[0]), min(br[0], heatmap_size[0])
484
- img_y = max(0, ul[1]), min(br[1], heatmap_size[1])
485
-
486
- v = target_weight[joint_id]
487
- if v > 0.5:
488
- target[joint_id][img_y[0]:img_y[1], img_x[0]:img_x[1]] = \
489
- g[g_y[0]:g_y[1], g_x[0]:g_x[1]]
490
-
491
- return target, target_weight
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /utils / streamer.py DELETED
@@ -1,142 +0,0 @@
1
- import cv2
2
- import torch
3
- import numpy as np
4
- import imageio
5
-
6
-
7
- def aug_matrix(w1, h1, w2, h2):
8
- dx = (w2 - w1) / 2.0
9
- dy = (h2 - h1) / 2.0
10
-
11
- matrix_trans = np.array([[1.0, 0, dx],
12
- [0, 1.0, dy],
13
- [0, 0, 1.0]])
14
-
15
- scale = np.min([float(w2)/w1, float(h2)/h1])
16
-
17
- M = get_affine_matrix(
18
- center=(w2 / 2.0, h2 / 2.0),
19
- translate=(0, 0),
20
- scale=scale)
21
-
22
- M = np.array(M + [0., 0., 1.]).reshape(3, 3)
23
- M = M.dot(matrix_trans)
24
-
25
- return M
26
-
27
-
28
- def get_affine_matrix(center, translate, scale):
29
- cx, cy = center
30
- tx, ty = translate
31
-
32
- M = [1, 0, 0,
33
- 0, 1, 0]
34
- M = [x * scale for x in M]
35
-
36
- # Apply translation and of center translation: RSS * C^-1
37
- M[2] += M[0] * (-cx) + M[1] * (-cy)
38
- M[5] += M[3] * (-cx) + M[4] * (-cy)
39
-
40
- # Apply center translation: T * C * RSS * C^-1
41
- M[2] += cx + tx
42
- M[5] += cy + ty
43
- return M
44
-
45
-
46
- class BaseStreamer():
47
- """This streamer will return images at 512x512 size.
48
- """
49
-
50
- def __init__(self,
51
- width=512, height=512, pad=True,
52
- mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5),
53
- **kwargs):
54
- self.width = width
55
- self.height = height
56
- self.pad = pad
57
- self.mean = np.array(mean)
58
- self.std = np.array(std)
59
-
60
- self.loader = self.create_loader()
61
-
62
- def create_loader(self):
63
- raise NotImplementedError
64
- yield np.zeros((600, 400, 3)) # in RGB (0, 255)
65
-
66
- def __getitem__(self, index):
67
- image = next(self.loader)
68
- in_height, in_width, _ = image.shape
69
- M = aug_matrix(in_width, in_height, self.width, self.height, self.pad)
70
- image = cv2.warpAffine(
71
- image, M[0:2, :], (self.width, self.height), flags=cv2.INTER_CUBIC)
72
-
73
- input = np.float32(image)
74
- input = (input / 255.0 - self.mean) / self.std # TO [-1.0, 1.0]
75
- input = input.transpose(2, 0, 1) # TO [3 x H x W]
76
- return torch.from_numpy(input).float()
77
-
78
- def __len__(self):
79
- raise NotImplementedError
80
-
81
-
82
- class CaptureStreamer(BaseStreamer):
83
- """This streamer takes webcam as input.
84
- """
85
-
86
- def __init__(self, id=0, width=512, height=512, pad=True, **kwargs):
87
- super().__init__(width, height, pad, **kwargs)
88
- self.capture = cv2.VideoCapture(id)
89
-
90
- def create_loader(self):
91
- while True:
92
- _, image = self.capture.read()
93
- image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # RGB
94
- yield image
95
-
96
- def __len__(self):
97
- return 100_000_000
98
-
99
- def __del__(self):
100
- self.capture.release()
101
-
102
-
103
- class VideoListStreamer(BaseStreamer):
104
- """This streamer takes a list of video files as input.
105
- """
106
-
107
- def __init__(self, files, width=512, height=512, pad=True, **kwargs):
108
- super().__init__(width, height, pad, **kwargs)
109
- self.files = files
110
- self.captures = [imageio.get_reader(f) for f in files]
111
- self.nframes = sum([int(cap._meta["fps"] * cap._meta["duration"])
112
- for cap in self.captures])
113
-
114
- def create_loader(self):
115
- for capture in self.captures:
116
- for image in capture: # RGB
117
- yield image
118
-
119
- def __len__(self):
120
- return self.nframes
121
-
122
- def __del__(self):
123
- for capture in self.captures:
124
- capture.close()
125
-
126
-
127
- class ImageListStreamer(BaseStreamer):
128
- """This streamer takes a list of image files as input.
129
- """
130
-
131
- def __init__(self, files, width=512, height=512, pad=True, **kwargs):
132
- super().__init__(width, height, pad, **kwargs)
133
- self.files = files
134
-
135
- def create_loader(self):
136
- for f in self.files:
137
- image = cv2.imread(f, cv2.IMREAD_UNCHANGED)[:, :, 0:3]
138
- image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # RGB
139
- yield image
140
-
141
- def __len__(self):
142
- return len(self.files)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / pymaf /utils /transforms.py DELETED
@@ -1,78 +0,0 @@
1
- # ------------------------------------------------------------------------------
2
- # Copyright (c) Microsoft
3
- # Licensed under the MIT License.
4
- # Written by Bin Xiao ([email protected])
5
- # ------------------------------------------------------------------------------
6
-
7
- from __future__ import absolute_import
8
- from __future__ import division
9
- from __future__ import print_function
10
-
11
- import cv2
12
- import numpy as np
13
-
14
-
15
- def transform_preds(coords, center, scale, output_size):
16
- target_coords = np.zeros(coords.shape)
17
- trans = get_affine_transform(center, scale, 0, output_size, inv=1)
18
- for p in range(coords.shape[0]):
19
- target_coords[p, 0:2] = affine_transform(coords[p, 0:2], trans)
20
- return target_coords
21
-
22
-
23
- def get_affine_transform(center,
24
- scale,
25
- rot,
26
- output_size,
27
- shift=np.array([0, 0], dtype=np.float32),
28
- inv=0):
29
- if not isinstance(scale, np.ndarray) and not isinstance(scale, list):
30
- # print(scale)
31
- scale = np.array([scale, scale])
32
-
33
- scale_tmp = scale * 200.0
34
- src_w = scale_tmp[0]
35
- dst_w = output_size[0]
36
- dst_h = output_size[1]
37
-
38
- rot_rad = np.pi * rot / 180
39
- src_dir = get_dir([0, src_w * -0.5], rot_rad)
40
- dst_dir = np.array([0, dst_w * -0.5], np.float32)
41
-
42
- src = np.zeros((3, 2), dtype=np.float32)
43
- dst = np.zeros((3, 2), dtype=np.float32)
44
- src[0, :] = center + scale_tmp * shift
45
- src[1, :] = center + src_dir + scale_tmp * shift
46
- dst[0, :] = [dst_w * 0.5, dst_h * 0.5]
47
- dst[1, :] = np.array([dst_w * 0.5, dst_h * 0.5]) + dst_dir
48
-
49
- src[2:, :] = get_3rd_point(src[0, :], src[1, :])
50
- dst[2:, :] = get_3rd_point(dst[0, :], dst[1, :])
51
-
52
- if inv:
53
- trans = cv2.getAffineTransform(np.float32(dst), np.float32(src))
54
- else:
55
- trans = cv2.getAffineTransform(np.float32(src), np.float32(dst))
56
-
57
- return trans
58
-
59
-
60
- def affine_transform(pt, t):
61
- new_pt = np.array([pt[0], pt[1], 1.]).T
62
- new_pt = np.dot(t, new_pt)
63
- return new_pt[:2]
64
-
65
-
66
- def get_3rd_point(a, b):
67
- direct = a - b
68
- return b + np.array([-direct[1], direct[0]], dtype=np.float32)
69
-
70
-
71
- def get_dir(src_point, rot_rad):
72
- sn, cs = np.sin(rot_rad), np.cos(rot_rad)
73
-
74
- src_result = [0, 0]
75
- src_result[0] = src_point[0] * cs - src_point[1] * sn
76
- src_result[1] = src_point[0] * sn + src_point[1] * cs
77
-
78
- return src_result
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / __init__.py DELETED
File without changes
lib / renderer / camera.py DELETED
@@ -1,226 +0,0 @@
1
-
2
- # -*- coding: utf-8 -*-
3
-
4
- # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
5
- # holder of all proprietary rights on this computer program.
6
- # You can only use this computer program if you have closed
7
- # a license agreement with MPG or you get the right to use the computer
8
- # program from someone who is authorized to grant you that right.
9
- # Any use of the computer program without a valid license is prohibited and
10
- # liable to prosecution.
11
- #
12
- # Copyright©2019 Max-Planck-Gesellschaft zur Förderung
13
- # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
14
- # for Intelligent Systems. All rights reserved.
15
- #
16
- # Contact: [email protected]
17
-
18
- import cv2
19
- import numpy as np
20
-
21
- from .glm import ortho
22
-
23
-
24
- class Camera:
25
- def __init__(self, width=1600, height=1200):
26
- # Focal Length
27
- # equivalent 50mm
28
- focal = np.sqrt(width * width + height * height)
29
- self.focal_x = focal
30
- self.focal_y = focal
31
- # Principal Point Offset
32
- self.principal_x = width / 2
33
- self.principal_y = height / 2
34
- # Axis Skew
35
- self.skew = 0
36
- # Image Size
37
- self.width = width
38
- self.height = height
39
-
40
- self.near = 1
41
- self.far = 10
42
-
43
- # Camera Center
44
- self.center = np.array([0, 0, 1.6])
45
- self.direction = np.array([0, 0, -1])
46
- self.right = np.array([1, 0, 0])
47
- self.up = np.array([0, 1, 0])
48
-
49
- self.ortho_ratio = None
50
-
51
- def sanity_check(self):
52
- self.center = self.center.reshape([-1])
53
- self.direction = self.direction.reshape([-1])
54
- self.right = self.right.reshape([-1])
55
- self.up = self.up.reshape([-1])
56
-
57
- assert len(self.center) == 3
58
- assert len(self.direction) == 3
59
- assert len(self.right) == 3
60
- assert len(self.up) == 3
61
-
62
- @staticmethod
63
- def normalize_vector(v):
64
- v_norm = np.linalg.norm(v)
65
- return v if v_norm == 0 else v / v_norm
66
-
67
- def get_real_z_value(self, z):
68
- z_near = self.near
69
- z_far = self.far
70
- z_n = 2.0 * z - 1.0
71
- z_e = 2.0 * z_near * z_far / (z_far + z_near - z_n * (z_far - z_near))
72
- return z_e
73
-
74
- def get_rotation_matrix(self):
75
- rot_mat = np.eye(3)
76
- s = self.right
77
- s = self.normalize_vector(s)
78
- rot_mat[0, :] = s
79
- u = self.up
80
- u = self.normalize_vector(u)
81
- rot_mat[1, :] = -u
82
- rot_mat[2, :] = self.normalize_vector(self.direction)
83
-
84
- return rot_mat
85
-
86
- def get_translation_vector(self):
87
- rot_mat = self.get_rotation_matrix()
88
- trans = -np.dot(rot_mat, self.center)
89
- return trans
90
-
91
- def get_intrinsic_matrix(self):
92
- int_mat = np.eye(3)
93
-
94
- int_mat[0, 0] = self.focal_x
95
- int_mat[1, 1] = self.focal_y
96
- int_mat[0, 1] = self.skew
97
- int_mat[0, 2] = self.principal_x
98
- int_mat[1, 2] = self.principal_y
99
-
100
- return int_mat
101
-
102
- def get_projection_matrix(self):
103
- ext_mat = self.get_extrinsic_matrix()
104
- int_mat = self.get_intrinsic_matrix()
105
-
106
- return np.matmul(int_mat, ext_mat)
107
-
108
- def get_extrinsic_matrix(self):
109
- rot_mat = self.get_rotation_matrix()
110
- int_mat = self.get_intrinsic_matrix()
111
- trans = self.get_translation_vector()
112
-
113
- extrinsic = np.eye(4)
114
- extrinsic[:3, :3] = rot_mat
115
- extrinsic[:3, 3] = trans
116
-
117
- return extrinsic[:3, :]
118
-
119
- def set_rotation_matrix(self, rot_mat):
120
- self.direction = rot_mat[2, :]
121
- self.up = -rot_mat[1, :]
122
- self.right = rot_mat[0, :]
123
-
124
- def set_intrinsic_matrix(self, int_mat):
125
- self.focal_x = int_mat[0, 0]
126
- self.focal_y = int_mat[1, 1]
127
- self.skew = int_mat[0, 1]
128
- self.principal_x = int_mat[0, 2]
129
- self.principal_y = int_mat[1, 2]
130
-
131
- def set_projection_matrix(self, proj_mat):
132
- res = cv2.decomposeProjectionMatrix(proj_mat)
133
- int_mat, rot_mat, camera_center_homo = res[0], res[1], res[2]
134
- camera_center = camera_center_homo[0:3] / camera_center_homo[3]
135
- camera_center = camera_center.reshape(-1)
136
- int_mat = int_mat / int_mat[2][2]
137
-
138
- self.set_intrinsic_matrix(int_mat)
139
- self.set_rotation_matrix(rot_mat)
140
- self.center = camera_center
141
-
142
- self.sanity_check()
143
-
144
- def get_gl_matrix(self):
145
- z_near = self.near
146
- z_far = self.far
147
- rot_mat = self.get_rotation_matrix()
148
- int_mat = self.get_intrinsic_matrix()
149
- trans = self.get_translation_vector()
150
-
151
- extrinsic = np.eye(4)
152
- extrinsic[:3, :3] = rot_mat
153
- extrinsic[:3, 3] = trans
154
- axis_adj = np.eye(4)
155
- axis_adj[2, 2] = -1
156
- axis_adj[1, 1] = -1
157
- model_view = np.matmul(axis_adj, extrinsic)
158
-
159
- projective = np.zeros([4, 4])
160
- projective[:2, :2] = int_mat[:2, :2]
161
- projective[:2, 2:3] = -int_mat[:2, 2:3]
162
- projective[3, 2] = -1
163
- projective[2, 2] = (z_near + z_far)
164
- projective[2, 3] = (z_near * z_far)
165
-
166
- if self.ortho_ratio is None:
167
- ndc = ortho(0, self.width, 0, self.height, z_near, z_far)
168
- perspective = np.matmul(ndc, projective)
169
- else:
170
- perspective = ortho(-self.width * self.ortho_ratio / 2,
171
- self.width * self.ortho_ratio / 2,
172
- -self.height * self.ortho_ratio / 2,
173
- self.height * self.ortho_ratio / 2, z_near,
174
- z_far)
175
-
176
- return perspective, model_view
177
-
178
-
179
- def KRT_from_P(proj_mat, normalize_K=True):
180
- res = cv2.decomposeProjectionMatrix(proj_mat)
181
- K, Rot, camera_center_homog = res[0], res[1], res[2]
182
- camera_center = camera_center_homog[0:3] / camera_center_homog[3]
183
- trans = -Rot.dot(camera_center)
184
- if normalize_K:
185
- K = K / K[2][2]
186
- return K, Rot, trans
187
-
188
-
189
- def MVP_from_P(proj_mat, width, height, near=0.1, far=10000):
190
- '''
191
- Convert OpenCV camera calibration matrix to OpenGL projection and model view matrix
192
- :param proj_mat: OpenCV camera projeciton matrix
193
- :param width: Image width
194
- :param height: Image height
195
- :param near: Z near value
196
- :param far: Z far value
197
- :return: OpenGL projection matrix and model view matrix
198
- '''
199
- res = cv2.decomposeProjectionMatrix(proj_mat)
200
- K, Rot, camera_center_homog = res[0], res[1], res[2]
201
- camera_center = camera_center_homog[0:3] / camera_center_homog[3]
202
- trans = -Rot.dot(camera_center)
203
- K = K / K[2][2]
204
-
205
- extrinsic = np.eye(4)
206
- extrinsic[:3, :3] = Rot
207
- extrinsic[:3, 3:4] = trans
208
- axis_adj = np.eye(4)
209
- axis_adj[2, 2] = -1
210
- axis_adj[1, 1] = -1
211
- model_view = np.matmul(axis_adj, extrinsic)
212
-
213
- zFar = far
214
- zNear = near
215
- projective = np.zeros([4, 4])
216
- projective[:2, :2] = K[:2, :2]
217
- projective[:2, 2:3] = -K[:2, 2:3]
218
- projective[3, 2] = -1
219
- projective[2, 2] = (zNear + zFar)
220
- projective[2, 3] = (zNear * zFar)
221
-
222
- ndc = ortho(0, width, 0, height, zNear, zFar)
223
-
224
- perspective = np.matmul(ndc, projective)
225
-
226
- return perspective, model_view
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / __init__.py DELETED
File without changes
lib / renderer / gl / data / color.fs DELETED
@@ -1,20 +0,0 @@
1
- #version 330 core
2
-
3
- layout (location = 0) out vec4 FragColor;
4
- layout (location = 1) out vec4 FragNormal;
5
- layout (location = 2) out vec4 FragDepth;
6
-
7
- in vec3 Color;
8
- in vec3 CamNormal;
9
- in vec3 depth;
10
-
11
-
12
- void main()
13
- {
14
- FragColor = vec4(Color,1.0);
15
-
16
- vec3 cam_norm_normalized = normalize(CamNormal);
17
- vec3 rgb = (cam_norm_normalized + 1.0) / 2.0;
18
- FragNormal = vec4(rgb, 1.0);
19
- FragDepth = vec4(depth.xyz, 1.0);
20
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / data /color.vs DELETED
@@ -1,29 +0,0 @@
1
- #version 330 core
2
-
3
- layout (location = 0) in vec3 a_Position;
4
- layout (location = 1) in vec3 a_Color;
5
- layout (location = 2) in vec3 a_Normal;
6
-
7
- out vec3 CamNormal;
8
- out vec3 CamPos;
9
- out vec3 Color;
10
- out vec3 depth;
11
-
12
-
13
- uniform mat3 RotMat;
14
- uniform mat4 NormMat;
15
- uniform mat4 ModelMat;
16
- uniform mat4 PerspMat;
17
-
18
- void main()
19
- {
20
- vec3 a_Position = (NormMat * vec4(a_Position,1.0)).xyz;
21
- gl_Position = PerspMat * ModelMat * vec4(RotMat * a_Position, 1.0);
22
- Color = a_Color;
23
-
24
- mat3 R = mat3(ModelMat) * RotMat;
25
- CamNormal = (R * a_Normal);
26
-
27
- depth = vec3(gl_Position.z / gl_Position.w);
28
-
29
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / data /normal.fs DELETED
@@ -1,12 +0,0 @@
1
- #version 330
2
-
3
- out vec4 FragColor;
4
-
5
- in vec3 CamNormal;
6
-
7
- void main()
8
- {
9
- vec3 cam_norm_normalized = normalize(CamNormal);
10
- vec3 rgb = (cam_norm_normalized + 1.0) / 2.0;
11
- FragColor = vec4(rgb, 1.0);
12
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / data /normal.vs DELETED
@@ -1,15 +0,0 @@
1
- #version 330
2
-
3
- layout (location = 0) in vec3 Position;
4
- layout (location = 1) in vec3 Normal;
5
-
6
- out vec3 CamNormal;
7
-
8
- uniform mat4 ModelMat;
9
- uniform mat4 PerspMat;
10
-
11
- void main()
12
- {
13
- gl_Position = PerspMat * ModelMat * vec4(Position, 1.0);
14
- CamNormal = (ModelMat * vec4(Normal, 0.0)).xyz;
15
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / data /prt.fs DELETED
@@ -1,157 +0,0 @@
1
- #version 330
2
-
3
- uniform vec3 SHCoeffs[9];
4
- uniform uint analytic;
5
-
6
- uniform uint hasNormalMap;
7
- uniform uint hasAlbedoMap;
8
-
9
- uniform sampler2D AlbedoMap;
10
- uniform sampler2D NormalMap;
11
-
12
- in VertexData {
13
- vec3 Position;
14
- vec3 Depth;
15
- vec3 ModelNormal;
16
- vec2 Texcoord;
17
- vec3 Tangent;
18
- vec3 Bitangent;
19
- vec3 PRT1;
20
- vec3 PRT2;
21
- vec3 PRT3;
22
- vec3 Label;
23
- } VertexIn;
24
-
25
- layout (location = 0) out vec4 FragColor;
26
- layout (location = 1) out vec4 FragNormal;
27
- layout (location = 2) out vec4 FragPosition;
28
- layout (location = 3) out vec4 FragAlbedo;
29
- layout (location = 4) out vec4 FragShading;
30
- layout (location = 5) out vec4 FragPRT1;
31
- layout (location = 6) out vec4 FragPRT2;
32
- // layout (location = 7) out vec4 FragPRT3;
33
- layout (location = 7) out vec4 FragLabel;
34
-
35
-
36
- vec4 gammaCorrection(vec4 vec, float g)
37
- {
38
- return vec4(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g), vec.w);
39
- }
40
-
41
- vec3 gammaCorrection(vec3 vec, float g)
42
- {
43
- return vec3(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g));
44
- }
45
-
46
- void evaluateH(vec3 n, out float H[9])
47
- {
48
- float c1 = 0.429043, c2 = 0.511664,
49
- c3 = 0.743125, c4 = 0.886227, c5 = 0.247708;
50
-
51
- H[0] = c4;
52
- H[1] = 2.0 * c2 * n[1];
53
- H[2] = 2.0 * c2 * n[2];
54
- H[3] = 2.0 * c2 * n[0];
55
- H[4] = 2.0 * c1 * n[0] * n[1];
56
- H[5] = 2.0 * c1 * n[1] * n[2];
57
- H[6] = c3 * n[2] * n[2] - c5;
58
- H[7] = 2.0 * c1 * n[2] * n[0];
59
- H[8] = c1 * (n[0] * n[0] - n[1] * n[1]);
60
- }
61
-
62
- vec3 evaluateLightingModel(vec3 normal)
63
- {
64
- float H[9];
65
- evaluateH(normal, H);
66
- vec3 res = vec3(0.0);
67
- for (int i = 0; i < 9; i++) {
68
- res += H[i] * SHCoeffs[i];
69
- }
70
- return res;
71
- }
72
-
73
- // nC: coarse geometry normal, nH: fine normal from normal map
74
- vec3 evaluateLightingModelHybrid(vec3 nC, vec3 nH, mat3 prt)
75
- {
76
- float HC[9], HH[9];
77
- evaluateH(nC, HC);
78
- evaluateH(nH, HH);
79
-
80
- vec3 res = vec3(0.0);
81
- vec3 shadow = vec3(0.0);
82
- vec3 unshadow = vec3(0.0);
83
- for(int i = 0; i < 3; ++i){
84
- for(int j = 0; j < 3; ++j){
85
- int id = i*3+j;
86
- res += HH[id]* SHCoeffs[id];
87
- shadow += prt[i][j] * SHCoeffs[id];
88
- unshadow += HC[id] * SHCoeffs[id];
89
- }
90
- }
91
- vec3 ratio = clamp(shadow/unshadow,0.0,1.0);
92
- res = ratio * res;
93
-
94
- return res;
95
- }
96
-
97
- vec3 evaluateLightingModelPRT(mat3 prt)
98
- {
99
- vec3 res = vec3(0.0);
100
- for(int i = 0; i < 3; ++i){
101
- for(int j = 0; j < 3; ++j){
102
- res += prt[i][j] * SHCoeffs[i*3+j];
103
- }
104
- }
105
-
106
- return res;
107
- }
108
-
109
- void main()
110
- {
111
- vec2 uv = VertexIn.Texcoord;
112
- vec3 nC = normalize(VertexIn.ModelNormal);
113
- vec3 nml = nC;
114
- mat3 prt = mat3(VertexIn.PRT1, VertexIn.PRT2, VertexIn.PRT3);
115
-
116
- if(hasAlbedoMap == uint(0))
117
- FragAlbedo = vec4(1.0);
118
- else
119
- FragAlbedo = texture(AlbedoMap, uv);//gammaCorrection(texture(AlbedoMap, uv), 1.0/2.2);
120
-
121
- if(hasNormalMap == uint(0))
122
- {
123
- if(analytic == uint(0))
124
- FragShading = vec4(evaluateLightingModelPRT(prt), 1.0f);
125
- else
126
- FragShading = vec4(evaluateLightingModel(nC), 1.0f);
127
- }
128
- else
129
- {
130
- vec3 n_tan = normalize(texture(NormalMap, uv).rgb*2.0-vec3(1.0));
131
-
132
- mat3 TBN = mat3(normalize(VertexIn.Tangent),normalize(VertexIn.Bitangent),nC);
133
- vec3 nH = normalize(TBN * n_tan);
134
-
135
- if(analytic == uint(0))
136
- FragShading = vec4(evaluateLightingModelHybrid(nC,nH,prt),1.0f);
137
- else
138
- FragShading = vec4(evaluateLightingModel(nH), 1.0f);
139
-
140
- nml = nH;
141
- }
142
-
143
- FragShading = gammaCorrection(FragShading, 2.2);
144
- FragColor = clamp(FragAlbedo * FragShading, 0.0, 1.0);
145
- FragNormal = vec4(0.5*(nml+vec3(1.0)), 1.0);
146
- FragPosition = vec4(VertexIn.Depth.xyz, 1.0);
147
- FragShading = vec4(clamp(0.5*FragShading.xyz, 0.0, 1.0),1.0);
148
- // FragColor = gammaCorrection(clamp(FragAlbedo * FragShading, 0.0, 1.0),2.2);
149
- // FragNormal = vec4(0.5*(nml+vec3(1.0)), 1.0);
150
- // FragPosition = vec4(VertexIn.Position,VertexIn.Depth.x);
151
- // FragShading = vec4(gammaCorrection(clamp(0.5*FragShading.xyz, 0.0, 1.0),2.2),1.0);
152
- // FragAlbedo = gammaCorrection(FragAlbedo,2.2);
153
- FragPRT1 = vec4(VertexIn.PRT1,1.0);
154
- FragPRT2 = vec4(VertexIn.PRT2,1.0);
155
- // FragPRT3 = vec4(VertexIn.PRT3,1.0);
156
- FragLabel = vec4(VertexIn.Label,1.0);
157
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / data /prt.vs DELETED
@@ -1,156 +0,0 @@
1
- #version 330
2
-
3
- layout (location = 0) in vec3 a_Position;
4
- layout (location = 1) in vec3 a_Normal;
5
- layout (location = 2) in vec2 a_TextureCoord;
6
- layout (location = 3) in vec3 a_Tangent;
7
- layout (location = 4) in vec3 a_Bitangent;
8
- layout (location = 5) in vec3 a_PRT1;
9
- layout (location = 6) in vec3 a_PRT2;
10
- layout (location = 7) in vec3 a_PRT3;
11
- layout (location = 8) in vec3 a_Label;
12
- out VertexData {
13
- vec3 Position;
14
- vec3 Depth;
15
- vec3 ModelNormal;
16
- vec2 Texcoord;
17
- vec3 Tangent;
18
- vec3 Bitangent;
19
- vec3 PRT1;
20
- vec3 PRT2;
21
- vec3 PRT3;
22
- vec3 Label;
23
- } VertexOut;
24
- uniform mat3 RotMat;
25
- uniform mat4 NormMat;
26
- uniform mat4 ModelMat;
27
- uniform mat4 PerspMat;
28
- float s_c3 = 0.94617469575; // (3*sqrt(5))/(4*sqrt(pi))
29
- float s_c4 = -0.31539156525;// (-sqrt(5))/(4*sqrt(pi))
30
- float s_c5 = 0.54627421529; // (sqrt(15))/(4*sqrt(pi))
31
- float s_c_scale = 1.0/0.91529123286551084;
32
- float s_c_scale_inv = 0.91529123286551084;
33
- float s_rc2 = 1.5853309190550713*s_c_scale;
34
- float s_c4_div_c3 = s_c4/s_c3;
35
- float s_c4_div_c3_x2 = (s_c4/s_c3)*2.0;
36
- float s_scale_dst2 = s_c3 * s_c_scale_inv;
37
- float s_scale_dst4 = s_c5 * s_c_scale_inv;
38
- void OptRotateBand0(float x[1], mat3 R, out float dst[1])
39
- {
40
- dst[0] = x[0];
41
- }
42
- // 9 multiplies
43
- void OptRotateBand1(float x[3], mat3 R, out float dst[3])
44
- {
45
- // derived from SlowRotateBand1
46
- dst[0] = ( R[1][1])*x[0] + (-R[1][2])*x[1] + ( R[1][0])*x[2];
47
- dst[1] = (-R[2][1])*x[0] + ( R[2][2])*x[1] + (-R[2][0])*x[2];
48
- dst[2] = ( R[0][1])*x[0] + (-R[0][2])*x[1] + ( R[0][0])*x[2];
49
- }
50
- // 48 multiplies
51
- void OptRotateBand2(float x[5], mat3 R, out float dst[5])
52
- {
53
- // Sparse matrix multiply
54
- float sh0 = x[3] + x[4] + x[4] - x[1];
55
- float sh1 = x[0] + s_rc2*x[2] + x[3] + x[4];
56
- float sh2 = x[0];
57
- float sh3 = -x[3];
58
- float sh4 = -x[1];
59
-
60
- // Rotations. R0 and R1 just use the raw matrix columns
61
- float r2x = R[0][0] + R[0][1];
62
- float r2y = R[1][0] + R[1][1];
63
- float r2z = R[2][0] + R[2][1];
64
-
65
- float r3x = R[0][0] + R[0][2];
66
- float r3y = R[1][0] + R[1][2];
67
- float r3z = R[2][0] + R[2][2];
68
-
69
- float r4x = R[0][1] + R[0][2];
70
- float r4y = R[1][1] + R[1][2];
71
- float r4z = R[2][1] + R[2][2];
72
-
73
- // dense matrix multiplication one column at a time
74
-
75
- // column 0
76
- float sh0_x = sh0 * R[0][0];
77
- float sh0_y = sh0 * R[1][0];
78
- float d0 = sh0_x * R[1][0];
79
- float d1 = sh0_y * R[2][0];
80
- float d2 = sh0 * (R[2][0] * R[2][0] + s_c4_div_c3);
81
- float d3 = sh0_x * R[2][0];
82
- float d4 = sh0_x * R[0][0] - sh0_y * R[1][0];
83
-
84
- // column 1
85
- float sh1_x = sh1 * R[0][2];
86
- float sh1_y = sh1 * R[1][2];
87
- d0 += sh1_x * R[1][2];
88
- d1 += sh1_y * R[2][2];
89
- d2 += sh1 * (R[2][2] * R[2][2] + s_c4_div_c3);
90
- d3 += sh1_x * R[2][2];
91
- d4 += sh1_x * R[0][2] - sh1_y * R[1][2];
92
-
93
- // column 2
94
- float sh2_x = sh2 * r2x;
95
- float sh2_y = sh2 * r2y;
96
- d0 += sh2_x * r2y;
97
- d1 += sh2_y * r2z;
98
- d2 += sh2 * (r2z * r2z + s_c4_div_c3_x2);
99
- d3 += sh2_x * r2z;
100
- d4 += sh2_x * r2x - sh2_y * r2y;
101
-
102
- // column 3
103
- float sh3_x = sh3 * r3x;
104
- float sh3_y = sh3 * r3y;
105
- d0 += sh3_x * r3y;
106
- d1 += sh3_y * r3z;
107
- d2 += sh3 * (r3z * r3z + s_c4_div_c3_x2);
108
- d3 += sh3_x * r3z;
109
- d4 += sh3_x * r3x - sh3_y * r3y;
110
-
111
- // column 4
112
- float sh4_x = sh4 * r4x;
113
- float sh4_y = sh4 * r4y;
114
- d0 += sh4_x * r4y;
115
- d1 += sh4_y * r4z;
116
- d2 += sh4 * (r4z * r4z + s_c4_div_c3_x2);
117
- d3 += sh4_x * r4z;
118
- d4 += sh4_x * r4x - sh4_y * r4y;
119
-
120
- // extra multipliers
121
- dst[0] = d0;
122
- dst[1] = -d1;
123
- dst[2] = d2 * s_scale_dst2;
124
- dst[3] = -d3;
125
- dst[4] = d4 * s_scale_dst4;
126
- }
127
- void main()
128
- {
129
- // normalization
130
- vec3 pos = (NormMat * vec4(a_Position,1.0)).xyz;
131
- mat3 R = mat3(ModelMat) * RotMat;
132
- VertexOut.ModelNormal = (R * a_Normal);
133
- VertexOut.Position = R * pos;
134
- VertexOut.Texcoord = a_TextureCoord;
135
- VertexOut.Tangent = (R * a_Tangent);
136
- VertexOut.Bitangent = (R * a_Bitangent);
137
- VertexOut.Label = a_Label;
138
- float PRT0, PRT1[3], PRT2[5];
139
- PRT0 = a_PRT1[0];
140
- PRT1[0] = a_PRT1[1];
141
- PRT1[1] = a_PRT1[2];
142
- PRT1[2] = a_PRT2[0];
143
- PRT2[0] = a_PRT2[1];
144
- PRT2[1] = a_PRT2[2];
145
- PRT2[2] = a_PRT3[0];
146
- PRT2[3] = a_PRT3[1];
147
- PRT2[4] = a_PRT3[2];
148
- OptRotateBand1(PRT1, R, PRT1);
149
- OptRotateBand2(PRT2, R, PRT2);
150
- VertexOut.PRT1 = vec3(PRT0,PRT1[0],PRT1[1]);
151
- VertexOut.PRT2 = vec3(PRT1[2],PRT2[0],PRT2[1]);
152
- VertexOut.PRT3 = vec3(PRT2[2],PRT2[3],PRT2[4]);
153
- gl_Position = PerspMat * ModelMat * vec4(RotMat * pos, 1.0);
154
-
155
- VertexOut.Depth = vec3(gl_Position.z / gl_Position.w);
156
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / data /prt_uv.fs DELETED
@@ -1,141 +0,0 @@
1
- #version 330
2
-
3
- uniform vec3 SHCoeffs[9];
4
- uniform uint analytic;
5
-
6
- uniform uint hasNormalMap;
7
- uniform uint hasAlbedoMap;
8
-
9
- uniform sampler2D AlbedoMap;
10
- uniform sampler2D NormalMap;
11
-
12
- in VertexData {
13
- vec3 Position;
14
- vec3 ModelNormal;
15
- vec3 CameraNormal;
16
- vec2 Texcoord;
17
- vec3 Tangent;
18
- vec3 Bitangent;
19
- vec3 PRT1;
20
- vec3 PRT2;
21
- vec3 PRT3;
22
- } VertexIn;
23
-
24
- layout (location = 0) out vec4 FragColor;
25
- layout (location = 1) out vec4 FragPosition;
26
- layout (location = 2) out vec4 FragNormal;
27
-
28
- vec4 gammaCorrection(vec4 vec, float g)
29
- {
30
- return vec4(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g), vec.w);
31
- }
32
-
33
- vec3 gammaCorrection(vec3 vec, float g)
34
- {
35
- return vec3(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g));
36
- }
37
-
38
- void evaluateH(vec3 n, out float H[9])
39
- {
40
- float c1 = 0.429043, c2 = 0.511664,
41
- c3 = 0.743125, c4 = 0.886227, c5 = 0.247708;
42
-
43
- H[0] = c4;
44
- H[1] = 2.0 * c2 * n[1];
45
- H[2] = 2.0 * c2 * n[2];
46
- H[3] = 2.0 * c2 * n[0];
47
- H[4] = 2.0 * c1 * n[0] * n[1];
48
- H[5] = 2.0 * c1 * n[1] * n[2];
49
- H[6] = c3 * n[2] * n[2] - c5;
50
- H[7] = 2.0 * c1 * n[2] * n[0];
51
- H[8] = c1 * (n[0] * n[0] - n[1] * n[1]);
52
- }
53
-
54
- vec3 evaluateLightingModel(vec3 normal)
55
- {
56
- float H[9];
57
- evaluateH(normal, H);
58
- vec3 res = vec3(0.0);
59
- for (int i = 0; i < 9; i++) {
60
- res += H[i] * SHCoeffs[i];
61
- }
62
- return res;
63
- }
64
-
65
- // nC: coarse geometry normal, nH: fine normal from normal map
66
- vec3 evaluateLightingModelHybrid(vec3 nC, vec3 nH, mat3 prt)
67
- {
68
- float HC[9], HH[9];
69
- evaluateH(nC, HC);
70
- evaluateH(nH, HH);
71
-
72
- vec3 res = vec3(0.0);
73
- vec3 shadow = vec3(0.0);
74
- vec3 unshadow = vec3(0.0);
75
- for(int i = 0; i < 3; ++i){
76
- for(int j = 0; j < 3; ++j){
77
- int id = i*3+j;
78
- res += HH[id]* SHCoeffs[id];
79
- shadow += prt[i][j] * SHCoeffs[id];
80
- unshadow += HC[id] * SHCoeffs[id];
81
- }
82
- }
83
- vec3 ratio = clamp(shadow/unshadow,0.0,1.0);
84
- res = ratio * res;
85
-
86
- return res;
87
- }
88
-
89
- vec3 evaluateLightingModelPRT(mat3 prt)
90
- {
91
- vec3 res = vec3(0.0);
92
- for(int i = 0; i < 3; ++i){
93
- for(int j = 0; j < 3; ++j){
94
- res += prt[i][j] * SHCoeffs[i*3+j];
95
- }
96
- }
97
-
98
- return res;
99
- }
100
-
101
- void main()
102
- {
103
- vec2 uv = VertexIn.Texcoord;
104
- vec3 nM = normalize(VertexIn.ModelNormal);
105
- vec3 nC = normalize(VertexIn.CameraNormal);
106
- vec3 nml = nC;
107
- mat3 prt = mat3(VertexIn.PRT1, VertexIn.PRT2, VertexIn.PRT3);
108
-
109
- vec4 albedo, shading;
110
- if(hasAlbedoMap == uint(0))
111
- albedo = vec4(1.0);
112
- else
113
- albedo = texture(AlbedoMap, uv);//gammaCorrection(texture(AlbedoMap, uv), 1.0/2.2);
114
-
115
- if(hasNormalMap == uint(0))
116
- {
117
- if(analytic == uint(0))
118
- shading = vec4(evaluateLightingModelPRT(prt), 1.0f);
119
- else
120
- shading = vec4(evaluateLightingModel(nC), 1.0f);
121
- }
122
- else
123
- {
124
- vec3 n_tan = normalize(texture(NormalMap, uv).rgb*2.0-vec3(1.0));
125
-
126
- mat3 TBN = mat3(normalize(VertexIn.Tangent),normalize(VertexIn.Bitangent),nC);
127
- vec3 nH = normalize(TBN * n_tan);
128
-
129
- if(analytic == uint(0))
130
- shading = vec4(evaluateLightingModelHybrid(nC,nH,prt),1.0f);
131
- else
132
- shading = vec4(evaluateLightingModel(nH), 1.0f);
133
-
134
- nml = nH;
135
- }
136
-
137
- shading = gammaCorrection(shading, 2.2);
138
- FragColor = clamp(albedo * shading, 0.0, 1.0);
139
- FragPosition = vec4(VertexIn.Position,1.0);
140
- FragNormal = vec4(0.5*(nM+vec3(1.0)),1.0);
141
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / data /prt_uv.vs DELETED
@@ -1,168 +0,0 @@
1
- #version 330
2
-
3
- layout (location = 0) in vec3 a_Position;
4
- layout (location = 1) in vec3 a_Normal;
5
- layout (location = 2) in vec2 a_TextureCoord;
6
- layout (location = 3) in vec3 a_Tangent;
7
- layout (location = 4) in vec3 a_Bitangent;
8
- layout (location = 5) in vec3 a_PRT1;
9
- layout (location = 6) in vec3 a_PRT2;
10
- layout (location = 7) in vec3 a_PRT3;
11
-
12
- out VertexData {
13
- vec3 Position;
14
- vec3 ModelNormal;
15
- vec3 CameraNormal;
16
- vec2 Texcoord;
17
- vec3 Tangent;
18
- vec3 Bitangent;
19
- vec3 PRT1;
20
- vec3 PRT2;
21
- vec3 PRT3;
22
- } VertexOut;
23
-
24
- uniform mat3 RotMat;
25
- uniform mat4 NormMat;
26
- uniform mat4 ModelMat;
27
- uniform mat4 PerspMat;
28
-
29
- #define pi 3.1415926535897932384626433832795
30
-
31
- float s_c3 = 0.94617469575; // (3*sqrt(5))/(4*sqrt(pi))
32
- float s_c4 = -0.31539156525;// (-sqrt(5))/(4*sqrt(pi))
33
- float s_c5 = 0.54627421529; // (sqrt(15))/(4*sqrt(pi))
34
-
35
- float s_c_scale = 1.0/0.91529123286551084;
36
- float s_c_scale_inv = 0.91529123286551084;
37
-
38
- float s_rc2 = 1.5853309190550713*s_c_scale;
39
- float s_c4_div_c3 = s_c4/s_c3;
40
- float s_c4_div_c3_x2 = (s_c4/s_c3)*2.0;
41
-
42
- float s_scale_dst2 = s_c3 * s_c_scale_inv;
43
- float s_scale_dst4 = s_c5 * s_c_scale_inv;
44
-
45
- void OptRotateBand0(float x[1], mat3 R, out float dst[1])
46
- {
47
- dst[0] = x[0];
48
- }
49
-
50
- // 9 multiplies
51
- void OptRotateBand1(float x[3], mat3 R, out float dst[3])
52
- {
53
- // derived from SlowRotateBand1
54
- dst[0] = ( R[1][1])*x[0] + (-R[1][2])*x[1] + ( R[1][0])*x[2];
55
- dst[1] = (-R[2][1])*x[0] + ( R[2][2])*x[1] + (-R[2][0])*x[2];
56
- dst[2] = ( R[0][1])*x[0] + (-R[0][2])*x[1] + ( R[0][0])*x[2];
57
- }
58
-
59
- // 48 multiplies
60
- void OptRotateBand2(float x[5], mat3 R, out float dst[5])
61
- {
62
- // Sparse matrix multiply
63
- float sh0 = x[3] + x[4] + x[4] - x[1];
64
- float sh1 = x[0] + s_rc2*x[2] + x[3] + x[4];
65
- float sh2 = x[0];
66
- float sh3 = -x[3];
67
- float sh4 = -x[1];
68
-
69
- // Rotations. R0 and R1 just use the raw matrix columns
70
- float r2x = R[0][0] + R[0][1];
71
- float r2y = R[1][0] + R[1][1];
72
- float r2z = R[2][0] + R[2][1];
73
-
74
- float r3x = R[0][0] + R[0][2];
75
- float r3y = R[1][0] + R[1][2];
76
- float r3z = R[2][0] + R[2][2];
77
-
78
- float r4x = R[0][1] + R[0][2];
79
- float r4y = R[1][1] + R[1][2];
80
- float r4z = R[2][1] + R[2][2];
81
-
82
- // dense matrix multiplication one column at a time
83
-
84
- // column 0
85
- float sh0_x = sh0 * R[0][0];
86
- float sh0_y = sh0 * R[1][0];
87
- float d0 = sh0_x * R[1][0];
88
- float d1 = sh0_y * R[2][0];
89
- float d2 = sh0 * (R[2][0] * R[2][0] + s_c4_div_c3);
90
- float d3 = sh0_x * R[2][0];
91
- float d4 = sh0_x * R[0][0] - sh0_y * R[1][0];
92
-
93
- // column 1
94
- float sh1_x = sh1 * R[0][2];
95
- float sh1_y = sh1 * R[1][2];
96
- d0 += sh1_x * R[1][2];
97
- d1 += sh1_y * R[2][2];
98
- d2 += sh1 * (R[2][2] * R[2][2] + s_c4_div_c3);
99
- d3 += sh1_x * R[2][2];
100
- d4 += sh1_x * R[0][2] - sh1_y * R[1][2];
101
-
102
- // column 2
103
- float sh2_x = sh2 * r2x;
104
- float sh2_y = sh2 * r2y;
105
- d0 += sh2_x * r2y;
106
- d1 += sh2_y * r2z;
107
- d2 += sh2 * (r2z * r2z + s_c4_div_c3_x2);
108
- d3 += sh2_x * r2z;
109
- d4 += sh2_x * r2x - sh2_y * r2y;
110
-
111
- // column 3
112
- float sh3_x = sh3 * r3x;
113
- float sh3_y = sh3 * r3y;
114
- d0 += sh3_x * r3y;
115
- d1 += sh3_y * r3z;
116
- d2 += sh3 * (r3z * r3z + s_c4_div_c3_x2);
117
- d3 += sh3_x * r3z;
118
- d4 += sh3_x * r3x - sh3_y * r3y;
119
-
120
- // column 4
121
- float sh4_x = sh4 * r4x;
122
- float sh4_y = sh4 * r4y;
123
- d0 += sh4_x * r4y;
124
- d1 += sh4_y * r4z;
125
- d2 += sh4 * (r4z * r4z + s_c4_div_c3_x2);
126
- d3 += sh4_x * r4z;
127
- d4 += sh4_x * r4x - sh4_y * r4y;
128
-
129
- // extra multipliers
130
- dst[0] = d0;
131
- dst[1] = -d1;
132
- dst[2] = d2 * s_scale_dst2;
133
- dst[3] = -d3;
134
- dst[4] = d4 * s_scale_dst4;
135
- }
136
-
137
- void main()
138
- {
139
- // normalization
140
- mat3 R = mat3(ModelMat) * RotMat;
141
- VertexOut.ModelNormal = a_Normal;
142
- VertexOut.CameraNormal = (R * a_Normal);
143
- VertexOut.Position = a_Position;
144
- VertexOut.Texcoord = a_TextureCoord;
145
- VertexOut.Tangent = (R * a_Tangent);
146
- VertexOut.Bitangent = (R * a_Bitangent);
147
- float PRT0, PRT1[3], PRT2[5];
148
- PRT0 = a_PRT1[0];
149
- PRT1[0] = a_PRT1[1];
150
- PRT1[1] = a_PRT1[2];
151
- PRT1[2] = a_PRT2[0];
152
- PRT2[0] = a_PRT2[1];
153
- PRT2[1] = a_PRT2[2];
154
- PRT2[2] = a_PRT3[0];
155
- PRT2[3] = a_PRT3[1];
156
- PRT2[4] = a_PRT3[2];
157
-
158
- OptRotateBand1(PRT1, R, PRT1);
159
- OptRotateBand2(PRT2, R, PRT2);
160
-
161
- VertexOut.PRT1 = vec3(PRT0,PRT1[0],PRT1[1]);
162
- VertexOut.PRT2 = vec3(PRT1[2],PRT2[0],PRT2[1]);
163
- VertexOut.PRT3 = vec3(PRT2[2],PRT2[3],PRT2[4]);
164
-
165
- gl_Position = vec4(a_TextureCoord, 0.0, 1.0) - vec4(0.5, 0.5, 0, 0);
166
- gl_Position[0] *= 2.0;
167
- gl_Position[1] *= 2.0;
168
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / data /quad.fs DELETED
@@ -1,11 +0,0 @@
1
- #version 330 core
2
- out vec4 FragColor;
3
-
4
- in vec2 TexCoord;
5
-
6
- uniform sampler2D screenTexture;
7
-
8
- void main()
9
- {
10
- FragColor = texture(screenTexture, TexCoord);
11
- }
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / data /quad.vs DELETED
@@ -1,11 +0,0 @@
1
- #version 330 core
2
- layout (location = 0) in vec2 aPos;
3
- layout (location = 1) in vec2 aTexCoord;
4
-
5
- out vec2 TexCoord;
6
-
7
- void main()
8
- {
9
- gl_Position = vec4(aPos.x, aPos.y, 0.0, 1.0);
10
- TexCoord = aTexCoord;
11
- }
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / framework.py DELETED
@@ -1,95 +0,0 @@
1
- # Mario Rosasco, 2016
2
- # adapted from framework.cpp, Copyright (C) 2010-2012 by Jason L. McKesson
3
- # This file is licensed under the MIT License.
4
- #
5
- # NB: Unlike in the framework.cpp organization, the main loop is contained
6
- # in the tutorial files, not in this framework file. Additionally, a copy of
7
- # this module file must exist in the same directory as the tutorial files
8
- # to be imported properly.
9
-
10
- import os
11
- from OpenGL.GL import *
12
-
13
-
14
- # Function that creates and compiles shaders according to the given type (a GL enum value) and
15
- # shader program (a file containing a GLSL program).
16
- def loadShader(shaderType, shaderFile):
17
- # check if file exists, get full path name
18
- strFilename = findFileOrThrow(shaderFile)
19
- shaderData = None
20
- with open(strFilename, 'r') as f:
21
- shaderData = f.read()
22
-
23
- shader = glCreateShader(shaderType)
24
- glShaderSource(
25
- shader,
26
- shaderData) # note that this is a simpler function call than in C
27
-
28
- # This shader compilation is more explicit than the one used in
29
- # framework.cpp, which relies on a glutil wrapper function.
30
- # This is made explicit here mainly to decrease dependence on pyOpenGL
31
- # utilities and wrappers, which docs caution may change in future versions.
32
- glCompileShader(shader)
33
-
34
- status = glGetShaderiv(shader, GL_COMPILE_STATUS)
35
- if status == GL_FALSE:
36
- # Note that getting the error log is much simpler in Python than in C/C++
37
- # and does not require explicit handling of the string buffer
38
- strInfoLog = glGetShaderInfoLog(shader)
39
- strShaderType = ""
40
- if shaderType is GL_VERTEX_SHADER:
41
- strShaderType = "vertex"
42
- elif shaderType is GL_GEOMETRY_SHADER:
43
- strShaderType = "geometry"
44
- elif shaderType is GL_FRAGMENT_SHADER:
45
- strShaderType = "fragment"
46
-
47
- print("Compilation failure for " + strShaderType + " shader:\n" +
48
- str(strInfoLog))
49
-
50
- return shader
51
-
52
-
53
- # Function that accepts a list of shaders, compiles them, and returns a handle to the compiled program
54
- def createProgram(shaderList):
55
- program = glCreateProgram()
56
-
57
- for shader in shaderList:
58
- glAttachShader(program, shader)
59
-
60
- glLinkProgram(program)
61
-
62
- status = glGetProgramiv(program, GL_LINK_STATUS)
63
- if status == GL_FALSE:
64
- # Note that getting the error log is much simpler in Python than in C/C++
65
- # and does not require explicit handling of the string buffer
66
- strInfoLog = glGetProgramInfoLog(program)
67
- print("Linker failure: \n" + str(strInfoLog))
68
-
69
- for shader in shaderList:
70
- glDetachShader(program, shader)
71
-
72
- return program
73
-
74
-
75
- # Helper function to locate and open the target file (passed in as a string).
76
- # Returns the full path to the file as a string.
77
- def findFileOrThrow(strBasename):
78
- # Keep constant names in C-style convention, for readability
79
- # when comparing to C(/C++) code.
80
- if os.path.isfile(strBasename):
81
- return strBasename
82
-
83
- LOCAL_FILE_DIR = "data" + os.sep
84
- GLOBAL_FILE_DIR = os.path.dirname(
85
- os.path.abspath(__file__)) + os.sep + "data" + os.sep
86
-
87
- strFilename = LOCAL_FILE_DIR + strBasename
88
- if os.path.isfile(strFilename):
89
- return strFilename
90
-
91
- strFilename = GLOBAL_FILE_DIR + strBasename
92
- if os.path.isfile(strFilename):
93
- return strFilename
94
-
95
- raise IOError('Could not find target file ' + strBasename)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / glcontext.py DELETED
@@ -1,136 +0,0 @@
1
- """Headless GPU-accelerated OpenGL context creation on Google Colaboratory.
2
-
3
- Typical usage:
4
-
5
- # Optional PyOpenGL configuratiopn can be done here.
6
- # import OpenGL
7
- # OpenGL.ERROR_CHECKING = True
8
-
9
- # 'glcontext' must be imported before any OpenGL.* API.
10
- from lucid.misc.gl.glcontext import create_opengl_context
11
-
12
- # Now it's safe to import OpenGL and EGL functions
13
- import OpenGL.GL as gl
14
-
15
- # create_opengl_context() creates a GL context that is attached to an
16
- # offscreen surface of the specified size. Note that rendering to buffers
17
- # of other sizes and formats is still possible with OpenGL Framebuffers.
18
- #
19
- # Users are expected to directly use the EGL API in case more advanced
20
- # context management is required.
21
- width, height = 640, 480
22
- create_opengl_context((width, height))
23
-
24
- # OpenGL context is available here.
25
-
26
- """
27
-
28
- from __future__ import print_function
29
-
30
- # pylint: disable=unused-import,g-import-not-at-top,g-statement-before-imports
31
-
32
- try:
33
- import OpenGL
34
- except:
35
- print('This module depends on PyOpenGL.')
36
- print('Please run "\033[1m!pip install -q pyopengl\033[0m" '
37
- 'prior importing this module.')
38
- raise
39
-
40
- import ctypes
41
- from ctypes import pointer, util
42
- import os
43
-
44
- os.environ['PYOPENGL_PLATFORM'] = 'egl'
45
-
46
- # OpenGL loading workaround.
47
- #
48
- # * PyOpenGL tries to load libGL, but we need libOpenGL, see [1,2].
49
- # This could have been solved by a symlink libGL->libOpenGL, but:
50
- #
51
- # * Python 2.7 can't find libGL and linEGL due to a bug (see [3])
52
- # in ctypes.util, that was only wixed in Python 3.6.
53
- #
54
- # So, the only solution I've found is to monkeypatch ctypes.util
55
- # [1] https://devblogs.nvidia.com/egl-eye-opengl-visualization-without-x-server/
56
- # [2] https://devblogs.nvidia.com/linking-opengl-server-side-rendering/
57
- # [3] https://bugs.python.org/issue9998
58
- _find_library_old = ctypes.util.find_library
59
- try:
60
-
61
- def _find_library_new(name):
62
- return {
63
- 'GL': 'libOpenGL.so',
64
- 'EGL': 'libEGL.so',
65
- }.get(name, _find_library_old(name))
66
-
67
- util.find_library = _find_library_new
68
- import OpenGL.GL as gl
69
- import OpenGL.EGL as egl
70
- except:
71
- print('Unable to load OpenGL libraries. '
72
- 'Make sure you use GPU-enabled backend.')
73
- print('Press "Runtime->Change runtime type" and set '
74
- '"Hardware accelerator" to GPU.')
75
- raise
76
- finally:
77
- util.find_library = _find_library_old
78
-
79
-
80
- def create_opengl_context(surface_size=(640, 480)):
81
- """Create offscreen OpenGL context and make it current.
82
-
83
- Users are expected to directly use EGL API in case more advanced
84
- context management is required.
85
-
86
- Args:
87
- surface_size: (width, height), size of the offscreen rendering surface.
88
- """
89
- egl_display = egl.eglGetDisplay(egl.EGL_DEFAULT_DISPLAY)
90
-
91
- major, minor = egl.EGLint(), egl.EGLint()
92
- egl.eglInitialize(egl_display, pointer(major), pointer(minor))
93
-
94
- config_attribs = [
95
- egl.EGL_SURFACE_TYPE, egl.EGL_PBUFFER_BIT, egl.EGL_BLUE_SIZE, 8,
96
- egl.EGL_GREEN_SIZE, 8, egl.EGL_RED_SIZE, 8, egl.EGL_DEPTH_SIZE, 24,
97
- egl.EGL_RENDERABLE_TYPE, egl.EGL_OPENGL_BIT, egl.EGL_NONE
98
- ]
99
- config_attribs = (egl.EGLint * len(config_attribs))(*config_attribs)
100
-
101
- num_configs = egl.EGLint()
102
- egl_cfg = egl.EGLConfig()
103
- egl.eglChooseConfig(egl_display, config_attribs, pointer(egl_cfg), 1,
104
- pointer(num_configs))
105
-
106
- width, height = surface_size
107
- pbuffer_attribs = [
108
- egl.EGL_WIDTH,
109
- width,
110
- egl.EGL_HEIGHT,
111
- height,
112
- egl.EGL_NONE,
113
- ]
114
- pbuffer_attribs = (egl.EGLint * len(pbuffer_attribs))(*pbuffer_attribs)
115
- egl_surf = egl.eglCreatePbufferSurface(egl_display, egl_cfg,
116
- pbuffer_attribs)
117
-
118
- egl.eglBindAPI(egl.EGL_OPENGL_API)
119
-
120
- context_attribs = None
121
- # context_attribs = [
122
- # egl.EGL_CONTEXT_MAJOR_VERSION,
123
- # 4,
124
- # egl.EGL_CONTEXT_MINOR_VERSION,
125
- # 1,
126
- # egl.EGL_NONE,
127
- # ]
128
-
129
- egl_context = egl.eglCreateContext(egl_display, egl_cfg,
130
- egl.EGL_NO_CONTEXT, context_attribs)
131
- egl.eglMakeCurrent(egl_display, egl_surf, egl_surf, egl_context)
132
-
133
- buffer_type = egl.EGLint()
134
- out = egl.eglQueryContext(egl_display, egl_context,
135
- egl.EGL_CONTEXT_CLIENT_VERSION, buffer_type)
136
- # print(buffer_type)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / init_gl.py DELETED
@@ -1,24 +0,0 @@
1
- _glut_window = None
2
- _context_inited = None
3
-
4
-
5
- def initialize_GL_context(width=512, height=512, egl=False):
6
- '''
7
- default context uses GLUT
8
- '''
9
- if not egl:
10
- import OpenGL.GLUT as GLUT
11
- display_mode = GLUT.GLUT_DOUBLE | GLUT.GLUT_RGB | GLUT.GLUT_DEPTH
12
- global _glut_window
13
- if _glut_window is None:
14
- GLUT.glutInit()
15
- GLUT.glutInitDisplayMode(display_mode)
16
- GLUT.glutInitWindowSize(width, height)
17
- GLUT.glutInitWindowPosition(0, 0)
18
- _glut_window = GLUT.glutCreateWindow("My Render.")
19
- else:
20
- from .glcontext import create_opengl_context
21
- global _context_inited
22
- if _context_inited is None:
23
- create_opengl_context((width, height))
24
- _context_inited = True
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / norm_render.py DELETED
@@ -1,75 +0,0 @@
1
- '''
2
- MIT License
3
- Copyright (c) 2019 Shunsuke Saito, Zeng Huang, and Ryota Natsume
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
- The above copyright notice and this permission notice shall be included in all
11
- copies or substantial portions of the Software.
12
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
13
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
14
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
15
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
16
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
17
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
18
- SOFTWARE.
19
- '''
20
- from OpenGL.GLUT import *
21
-
22
- from .render2 import Render
23
-
24
-
25
- class NormRender(Render):
26
- def __init__(self,
27
- width=1600,
28
- height=1200,
29
- name='Cam Renderer',
30
- program_files=['simple.fs', 'simple.vs'],
31
- color_size=1,
32
- ms_rate=1):
33
- Render.__init__(self, width, height, name, program_files, color_size,
34
- ms_rate)
35
- self.camera = None
36
-
37
- glutDisplayFunc(self.display)
38
- glutKeyboardFunc(self.keyboard)
39
-
40
- def set_camera(self, camera):
41
- self.camera = camera
42
- self.projection_matrix, self.model_view_matrix = camera.get_gl_matrix()
43
-
44
- def set_matrices(self, projection, modelview):
45
- self.projection_matrix = projection
46
- self.model_view_matrix = modelview
47
-
48
- def keyboard(self, key, x, y):
49
- # up
50
- eps = 1
51
- # print(key)
52
- if key == b'w':
53
- self.camera.center += eps * self.camera.direction
54
- elif key == b's':
55
- self.camera.center -= eps * self.camera.direction
56
- if key == b'a':
57
- self.camera.center -= eps * self.camera.right
58
- elif key == b'd':
59
- self.camera.center += eps * self.camera.right
60
- if key == b' ':
61
- self.camera.center += eps * self.camera.up
62
- elif key == b'x':
63
- self.camera.center -= eps * self.camera.up
64
- elif key == b'i':
65
- self.camera.near += 0.1 * eps
66
- self.camera.far += 0.1 * eps
67
- elif key == b'o':
68
- self.camera.near -= 0.1 * eps
69
- self.camera.far -= 0.1 * eps
70
-
71
- self.projection_matrix, self.model_view_matrix = self.camera.get_gl_matrix(
72
- )
73
-
74
- def show(self):
75
- glutMainLoop()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl / render.py DELETED
@@ -1,380 +0,0 @@
1
-
2
- # -*- coding: utf-8 -*-
3
-
4
- # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
5
- # holder of all proprietary rights on this computer program.
6
- # You can only use this computer program if you have closed
7
- # a license agreement with MPG or you get the right to use the computer
8
- # program from someone who is authorized to grant you that right.
9
- # Any use of the computer program without a valid license is prohibited and
10
- # liable to prosecution.
11
- #
12
- # Copyright©2019 Max-Planck-Gesellschaft zur Förderung
13
- # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
14
- # for Intelligent Systems. All rights reserved.
15
- #
16
- # Contact: [email protected]
17
-
18
- from ctypes import *
19
-
20
- import numpy as np
21
- from .framework import *
22
-
23
- GLUT = None
24
-
25
-
26
- # NOTE: Render class assumes GL context is created already.
27
- class Render:
28
- def __init__(self,
29
- width=1600,
30
- height=1200,
31
- name='GL Renderer',
32
- program_files=['simple.fs', 'simple.vs'],
33
- color_size=1,
34
- ms_rate=1,
35
- egl=False):
36
- self.width = width
37
- self.height = height
38
- self.name = name
39
- self.use_inverse_depth = False
40
- self.egl = egl
41
-
42
- glEnable(GL_DEPTH_TEST)
43
-
44
- glClampColor(GL_CLAMP_READ_COLOR, GL_FALSE)
45
- glClampColor(GL_CLAMP_FRAGMENT_COLOR, GL_FALSE)
46
- glClampColor(GL_CLAMP_VERTEX_COLOR, GL_FALSE)
47
-
48
- # init program
49
- shader_list = []
50
-
51
- for program_file in program_files:
52
- _, ext = os.path.splitext(program_file)
53
- if ext == '.vs':
54
- shader_list.append(loadShader(GL_VERTEX_SHADER, program_file))
55
- elif ext == '.fs':
56
- shader_list.append(loadShader(GL_FRAGMENT_SHADER,
57
- program_file))
58
- elif ext == '.gs':
59
- shader_list.append(loadShader(GL_GEOMETRY_SHADER,
60
- program_file))
61
-
62
- self.program = createProgram(shader_list)
63
-
64
- for shader in shader_list:
65
- glDeleteShader(shader)
66
-
67
- # Init uniform variables
68
- self.model_mat_unif = glGetUniformLocation(self.program, 'ModelMat')
69
- self.persp_mat_unif = glGetUniformLocation(self.program, 'PerspMat')
70
-
71
- self.vertex_buffer = glGenBuffers(1)
72
-
73
- # Init screen quad program and buffer
74
- self.quad_program, self.quad_buffer = self.init_quad_program()
75
-
76
- # Configure frame buffer
77
- self.frame_buffer = glGenFramebuffers(1)
78
- glBindFramebuffer(GL_FRAMEBUFFER, self.frame_buffer)
79
-
80
- self.intermediate_fbo = None
81
- if ms_rate > 1:
82
- # Configure texture buffer to render to
83
- self.color_buffer = []
84
- for i in range(color_size):
85
- color_buffer = glGenTextures(1)
86
- multi_sample_rate = ms_rate
87
- glBindTexture(GL_TEXTURE_2D_MULTISAMPLE, color_buffer)
88
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,
89
- GL_CLAMP_TO_EDGE)
90
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,
91
- GL_CLAMP_TO_EDGE)
92
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
93
- GL_LINEAR)
94
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
95
- GL_LINEAR)
96
- glTexImage2DMultisample(GL_TEXTURE_2D_MULTISAMPLE,
97
- multi_sample_rate, GL_RGBA32F,
98
- self.width, self.height, GL_TRUE)
99
- glBindTexture(GL_TEXTURE_2D_MULTISAMPLE, 0)
100
- glFramebufferTexture2D(GL_FRAMEBUFFER,
101
- GL_COLOR_ATTACHMENT0 + i,
102
- GL_TEXTURE_2D_MULTISAMPLE, color_buffer,
103
- 0)
104
- self.color_buffer.append(color_buffer)
105
-
106
- self.render_buffer = glGenRenderbuffers(1)
107
- glBindRenderbuffer(GL_RENDERBUFFER, self.render_buffer)
108
- glRenderbufferStorageMultisample(GL_RENDERBUFFER,
109
- multi_sample_rate,
110
- GL_DEPTH24_STENCIL8, self.width,
111
- self.height)
112
- glBindRenderbuffer(GL_RENDERBUFFER, 0)
113
- glFramebufferRenderbuffer(GL_FRAMEBUFFER,
114
- GL_DEPTH_STENCIL_ATTACHMENT,
115
- GL_RENDERBUFFER, self.render_buffer)
116
-
117
- attachments = []
118
- for i in range(color_size):
119
- attachments.append(GL_COLOR_ATTACHMENT0 + i)
120
- glDrawBuffers(color_size, attachments)
121
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
122
-
123
- self.intermediate_fbo = glGenFramebuffers(1)
124
- glBindFramebuffer(GL_FRAMEBUFFER, self.intermediate_fbo)
125
-
126
- self.screen_texture = []
127
- for i in range(color_size):
128
- screen_texture = glGenTextures(1)
129
- glBindTexture(GL_TEXTURE_2D, screen_texture)
130
- glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, self.width,
131
- self.height, 0, GL_RGBA, GL_FLOAT, None)
132
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
133
- GL_LINEAR)
134
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
135
- GL_LINEAR)
136
- glFramebufferTexture2D(GL_FRAMEBUFFER,
137
- GL_COLOR_ATTACHMENT0 + i, GL_TEXTURE_2D,
138
- screen_texture, 0)
139
- self.screen_texture.append(screen_texture)
140
-
141
- glDrawBuffers(color_size, attachments)
142
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
143
- else:
144
- self.color_buffer = []
145
- for i in range(color_size):
146
- color_buffer = glGenTextures(1)
147
- glBindTexture(GL_TEXTURE_2D, color_buffer)
148
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,
149
- GL_CLAMP_TO_EDGE)
150
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,
151
- GL_CLAMP_TO_EDGE)
152
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
153
- GL_NEAREST)
154
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
155
- GL_NEAREST)
156
- glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, self.width,
157
- self.height, 0, GL_RGBA, GL_FLOAT, None)
158
- glFramebufferTexture2D(GL_FRAMEBUFFER,
159
- GL_COLOR_ATTACHMENT0 + i, GL_TEXTURE_2D,
160
- color_buffer, 0)
161
- self.color_buffer.append(color_buffer)
162
-
163
- # Configure depth texture map to render to
164
- self.depth_buffer = glGenTextures(1)
165
- glBindTexture(GL_TEXTURE_2D, self.depth_buffer)
166
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
167
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)
168
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
169
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
170
- glTexParameteri(GL_TEXTURE_2D, GL_DEPTH_TEXTURE_MODE, GL_INTENSITY)
171
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_MODE,
172
- GL_COMPARE_R_TO_TEXTURE)
173
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FUNC, GL_LEQUAL)
174
- glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT, self.width,
175
- self.height, 0, GL_DEPTH_COMPONENT, GL_FLOAT, None)
176
- glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT,
177
- GL_TEXTURE_2D, self.depth_buffer, 0)
178
-
179
- attachments = []
180
- for i in range(color_size):
181
- attachments.append(GL_COLOR_ATTACHMENT0 + i)
182
- glDrawBuffers(color_size, attachments)
183
- self.screen_texture = self.color_buffer
184
-
185
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
186
-
187
- # Configure texture buffer if needed
188
- self.render_texture = None
189
-
190
- # NOTE: original render_texture only support one input
191
- # this is tentative member of this issue
192
- self.render_texture_v2 = {}
193
-
194
- # Inner storage for buffer data
195
- self.vertex_data = None
196
- self.vertex_dim = None
197
- self.n_vertices = None
198
-
199
- self.model_view_matrix = None
200
- self.projection_matrix = None
201
-
202
- if not egl:
203
- global GLUT
204
- import OpenGL.GLUT as GLUT
205
- GLUT.glutDisplayFunc(self.display)
206
-
207
- def init_quad_program(self):
208
- shader_list = []
209
-
210
- shader_list.append(loadShader(GL_VERTEX_SHADER, "quad.vs"))
211
- shader_list.append(loadShader(GL_FRAGMENT_SHADER, "quad.fs"))
212
-
213
- the_program = createProgram(shader_list)
214
-
215
- for shader in shader_list:
216
- glDeleteShader(shader)
217
-
218
- # vertex attributes for a quad that fills the entire screen in Normalized Device Coordinates.
219
- # positions # texCoords
220
- quad_vertices = np.array([
221
- -1.0, 1.0, 0.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0, -1.0, 1.0, 0.0,
222
- -1.0, 1.0, 0.0, 1.0, 1.0, -1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0
223
- ])
224
-
225
- quad_buffer = glGenBuffers(1)
226
- glBindBuffer(GL_ARRAY_BUFFER, quad_buffer)
227
- glBufferData(GL_ARRAY_BUFFER, quad_vertices, GL_STATIC_DRAW)
228
-
229
- glBindBuffer(GL_ARRAY_BUFFER, 0)
230
-
231
- return the_program, quad_buffer
232
-
233
- def set_mesh(self, vertices, faces):
234
- self.vertex_data = vertices[faces.reshape([-1])]
235
- self.vertex_dim = self.vertex_data.shape[1]
236
- self.n_vertices = self.vertex_data.shape[0]
237
-
238
- glBindBuffer(GL_ARRAY_BUFFER, self.vertex_buffer)
239
- glBufferData(GL_ARRAY_BUFFER, self.vertex_data, GL_STATIC_DRAW)
240
-
241
- glBindBuffer(GL_ARRAY_BUFFER, 0)
242
-
243
- def set_viewpoint(self, projection, model_view):
244
- self.projection_matrix = projection
245
- self.model_view_matrix = model_view
246
-
247
- def draw_init(self):
248
- glBindFramebuffer(GL_FRAMEBUFFER, self.frame_buffer)
249
- glEnable(GL_DEPTH_TEST)
250
-
251
- glClearColor(0.0, 0.0, 0.0, 0.0)
252
- if self.use_inverse_depth:
253
- glDepthFunc(GL_GREATER)
254
- glClearDepth(0.0)
255
- else:
256
- glDepthFunc(GL_LESS)
257
- glClearDepth(1.0)
258
- glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
259
-
260
- def draw_end(self):
261
- if self.intermediate_fbo is not None:
262
- for i in range(len(self.color_buffer)):
263
- glBindFramebuffer(GL_READ_FRAMEBUFFER, self.frame_buffer)
264
- glReadBuffer(GL_COLOR_ATTACHMENT0 + i)
265
- glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self.intermediate_fbo)
266
- glDrawBuffer(GL_COLOR_ATTACHMENT0 + i)
267
- glBlitFramebuffer(0, 0, self.width, self.height, 0, 0,
268
- self.width, self.height, GL_COLOR_BUFFER_BIT,
269
- GL_NEAREST)
270
-
271
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
272
- glDepthFunc(GL_LESS)
273
- glClearDepth(1.0)
274
-
275
- def draw(self):
276
- self.draw_init()
277
-
278
- glUseProgram(self.program)
279
- glUniformMatrix4fv(self.model_mat_unif, 1, GL_FALSE,
280
- self.model_view_matrix.transpose())
281
- glUniformMatrix4fv(self.persp_mat_unif, 1, GL_FALSE,
282
- self.projection_matrix.transpose())
283
-
284
- glBindBuffer(GL_ARRAY_BUFFER, self.vertex_buffer)
285
-
286
- glEnableVertexAttribArray(0)
287
- glVertexAttribPointer(0, self.vertex_dim, GL_DOUBLE, GL_FALSE, 0, None)
288
-
289
- glDrawArrays(GL_TRIANGLES, 0, self.n_vertices)
290
-
291
- glDisableVertexAttribArray(0)
292
-
293
- glBindBuffer(GL_ARRAY_BUFFER, 0)
294
-
295
- glUseProgram(0)
296
-
297
- self.draw_end()
298
-
299
- def get_color(self, color_id=0):
300
- glBindFramebuffer(
301
- GL_FRAMEBUFFER, self.intermediate_fbo
302
- if self.intermediate_fbo is not None else self.frame_buffer)
303
- glReadBuffer(GL_COLOR_ATTACHMENT0 + color_id)
304
- data = glReadPixels(0,
305
- 0,
306
- self.width,
307
- self.height,
308
- GL_RGBA,
309
- GL_FLOAT,
310
- outputType=None)
311
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
312
- rgb = data.reshape(self.height, self.width, -1)
313
- rgb = np.flip(rgb, 0)
314
- return rgb
315
-
316
- def get_z_value(self):
317
- glBindFramebuffer(GL_FRAMEBUFFER, self.frame_buffer)
318
- data = glReadPixels(0,
319
- 0,
320
- self.width,
321
- self.height,
322
- GL_DEPTH_COMPONENT,
323
- GL_FLOAT,
324
- outputType=None)
325
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
326
- z = data.reshape(self.height, self.width)
327
- z = np.flip(z, 0)
328
- return z
329
-
330
- def display(self):
331
- self.draw()
332
-
333
- if not self.egl:
334
- # First we draw a scene.
335
- # Notice the result is stored in the texture buffer.
336
-
337
- # Then we return to the default frame buffer since we will display on the screen.
338
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
339
-
340
- # Do the clean-up.
341
- glClearColor(0.0, 0.0, 0.0, 0.0)
342
- glClear(GL_COLOR_BUFFER_BIT)
343
-
344
- # We draw a rectangle which covers the whole screen.
345
- glUseProgram(self.quad_program)
346
- glBindBuffer(GL_ARRAY_BUFFER, self.quad_buffer)
347
-
348
- size_of_double = 8
349
- glEnableVertexAttribArray(0)
350
- glVertexAttribPointer(0, 2, GL_DOUBLE, GL_FALSE,
351
- 4 * size_of_double, None)
352
- glEnableVertexAttribArray(1)
353
- glVertexAttribPointer(1, 2, GL_DOUBLE, GL_FALSE,
354
- 4 * size_of_double,
355
- c_void_p(2 * size_of_double))
356
-
357
- glDisable(GL_DEPTH_TEST)
358
-
359
- # The stored texture is then mapped to this rectangle.
360
- # properly assing color buffer texture
361
- glActiveTexture(GL_TEXTURE0)
362
- glBindTexture(GL_TEXTURE_2D, self.screen_texture[0])
363
- glUniform1i(
364
- glGetUniformLocation(self.quad_program, 'screenTexture'), 0)
365
-
366
- glDrawArrays(GL_TRIANGLES, 0, 6)
367
-
368
- glDisableVertexAttribArray(1)
369
- glDisableVertexAttribArray(0)
370
-
371
- glEnable(GL_DEPTH_TEST)
372
- glBindBuffer(GL_ARRAY_BUFFER, 0)
373
- glUseProgram(0)
374
-
375
- GLUT.glutSwapBuffers()
376
- GLUT.glutPostRedisplay()
377
-
378
- def show(self):
379
- if not self.egl:
380
- GLUT.glutMainLoop()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl /cam_render.py DELETED
@@ -1,80 +0,0 @@
1
-
2
- # -*- coding: utf-8 -*-
3
-
4
- # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
5
- # holder of all proprietary rights on this computer program.
6
- # You can only use this computer program if you have closed
7
- # a license agreement with MPG or you get the right to use the computer
8
- # program from someone who is authorized to grant you that right.
9
- # Any use of the computer program without a valid license is prohibited and
10
- # liable to prosecution.
11
- #
12
- # Copyright©2019 Max-Planck-Gesellschaft zur Förderung
13
- # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
14
- # for Intelligent Systems. All rights reserved.
15
- #
16
- # Contact: [email protected]
17
-
18
- from .render import Render
19
-
20
- GLUT = None
21
-
22
-
23
- class CamRender(Render):
24
- def __init__(self,
25
- width=1600,
26
- height=1200,
27
- name='Cam Renderer',
28
- program_files=['simple.fs', 'simple.vs'],
29
- color_size=1,
30
- ms_rate=1,
31
- egl=False):
32
- Render.__init__(self,
33
- width,
34
- height,
35
- name,
36
- program_files,
37
- color_size,
38
- ms_rate=ms_rate,
39
- egl=egl)
40
- self.camera = None
41
-
42
- if not egl:
43
- global GLUT
44
- import OpenGL.GLUT as GLUT
45
- GLUT.glutDisplayFunc(self.display)
46
- GLUT.glutKeyboardFunc(self.keyboard)
47
-
48
- def set_camera(self, camera):
49
- self.camera = camera
50
- self.projection_matrix, self.model_view_matrix = camera.get_gl_matrix()
51
-
52
- def keyboard(self, key, x, y):
53
- # up
54
- eps = 1
55
- # print(key)
56
- if key == b'w':
57
- self.camera.center += eps * self.camera.direction
58
- elif key == b's':
59
- self.camera.center -= eps * self.camera.direction
60
- if key == b'a':
61
- self.camera.center -= eps * self.camera.right
62
- elif key == b'd':
63
- self.camera.center += eps * self.camera.right
64
- if key == b' ':
65
- self.camera.center += eps * self.camera.up
66
- elif key == b'x':
67
- self.camera.center -= eps * self.camera.up
68
- elif key == b'i':
69
- self.camera.near += 0.1 * eps
70
- self.camera.far += 0.1 * eps
71
- elif key == b'o':
72
- self.camera.near -= 0.1 * eps
73
- self.camera.far -= 0.1 * eps
74
-
75
- self.projection_matrix, self.model_view_matrix = self.camera.get_gl_matrix(
76
- )
77
-
78
- def show(self):
79
- if GLUT is not None:
80
- GLUT.glutMainLoop()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl /color_render.py DELETED
@@ -1,158 +0,0 @@
1
- '''
2
- MIT License
3
- Copyright (c) 2019 Shunsuke Saito, Zeng Huang, and Ryota Natsume
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
- The above copyright notice and this permission notice shall be included in all
11
- copies or substantial portions of the Software.
12
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
13
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
14
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
15
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
16
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
17
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
18
- SOFTWARE.
19
- '''
20
- import numpy as np
21
- import random
22
-
23
- from .framework import *
24
- from .cam_render import CamRender
25
-
26
-
27
- class ColorRender(CamRender):
28
- def __init__(self, width=1600, height=1200, name='Color Renderer', egl=False):
29
- program_files = ['color.vs', 'color.fs']
30
- CamRender.__init__(self,
31
- width,
32
- height,
33
- name,
34
- program_files=program_files,
35
- color_size=3, egl=egl)
36
-
37
- # WARNING: this differs from vertex_buffer and vertex_data in Render
38
- self.vert_buffer = {}
39
- self.vert_data = {}
40
-
41
- # normal
42
- self.norm_buffer = {}
43
- self.norm_data = {}
44
-
45
- self.color_buffer = {}
46
- self.color_data = {}
47
-
48
- self.vertex_dim = {}
49
- self.n_vertices = {}
50
-
51
- self.rot_mat_unif = glGetUniformLocation(self.program, 'RotMat')
52
- self.rot_matrix = np.eye(3)
53
-
54
- self.norm_mat_unif = glGetUniformLocation(self.program, 'NormMat')
55
- self.normalize_matrix = np.eye(4)
56
-
57
- def set_norm_mat(self, scale, center):
58
- N = np.eye(4)
59
- N[:3, :3] = scale * np.eye(3)
60
- N[:3, 3] = -scale * center
61
-
62
- self.normalize_matrix = N
63
-
64
- def set_mesh(self, vertices, faces, color, normals, mat_name='all'):
65
-
66
- self.vert_data[mat_name] = vertices[faces.reshape([-1])]
67
- self.n_vertices[mat_name] = self.vert_data[mat_name].shape[0]
68
- self.vertex_dim[mat_name] = self.vert_data[mat_name].shape[1]
69
- self.color_data[mat_name] = color[faces.reshape([-1])]
70
- self.norm_data[mat_name] = normals[faces.reshape([-1])]
71
-
72
- if mat_name not in self.vert_buffer.keys():
73
- self.vert_buffer[mat_name] = glGenBuffers(1)
74
- glBindBuffer(GL_ARRAY_BUFFER, self.vert_buffer[mat_name])
75
- glBufferData(GL_ARRAY_BUFFER, self.vert_data[mat_name], GL_STATIC_DRAW)
76
-
77
- if mat_name not in self.color_buffer.keys():
78
- self.color_buffer[mat_name] = glGenBuffers(1)
79
- glBindBuffer(GL_ARRAY_BUFFER, self.color_buffer[mat_name])
80
- glBufferData(GL_ARRAY_BUFFER, self.color_data[mat_name],
81
- GL_STATIC_DRAW)
82
-
83
- if mat_name not in self.norm_buffer.keys():
84
- self.norm_buffer[mat_name] = glGenBuffers(1)
85
- glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer[mat_name])
86
- glBufferData(GL_ARRAY_BUFFER, self.norm_data[mat_name], GL_STATIC_DRAW)
87
-
88
- glBindBuffer(GL_ARRAY_BUFFER, 0)
89
-
90
- def cleanup(self):
91
-
92
- glBindBuffer(GL_ARRAY_BUFFER, 0)
93
-
94
- for key in self.vert_data:
95
- glDeleteBuffers(1, [self.vert_buffer[key]])
96
- glDeleteBuffers(1, [self.color_buffer[key]])
97
- glDeleteBuffers(1, [self.norm_buffer[key]])
98
-
99
- self.norm_buffer = {}
100
- self.norm_data = {}
101
-
102
- self.vert_buffer = {}
103
- self.vert_data = {}
104
-
105
- self.color_buffer = {}
106
- self.color_data = {}
107
-
108
- self.render_texture_mat = {}
109
-
110
- self.vertex_dim = {}
111
- self.n_vertices = {}
112
-
113
- def draw(self):
114
- self.draw_init()
115
-
116
- glEnable(GL_MULTISAMPLE)
117
-
118
- glUseProgram(self.program)
119
- glUniformMatrix4fv(self.norm_mat_unif, 1, GL_FALSE,
120
- self.normalize_matrix.transpose())
121
- glUniformMatrix4fv(self.model_mat_unif, 1, GL_FALSE,
122
- self.model_view_matrix.transpose())
123
- glUniformMatrix4fv(self.persp_mat_unif, 1, GL_FALSE,
124
- self.projection_matrix.transpose())
125
- glUniformMatrix3fv(self.rot_mat_unif, 1, GL_FALSE,
126
- self.rot_matrix.transpose())
127
-
128
- for mat in self.vert_buffer:
129
-
130
- # Handle vertex buffer
131
- glBindBuffer(GL_ARRAY_BUFFER, self.vert_buffer[mat])
132
- glEnableVertexAttribArray(0)
133
- glVertexAttribPointer(0, self.vertex_dim[mat], GL_DOUBLE, GL_FALSE,
134
- 0, None)
135
-
136
- # Handle color buffer
137
- glBindBuffer(GL_ARRAY_BUFFER, self.color_buffer[mat])
138
- glEnableVertexAttribArray(1)
139
- glVertexAttribPointer(1, 3, GL_DOUBLE, GL_FALSE, 0, None)
140
-
141
- # Handle normal buffer
142
- glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer[mat])
143
- glEnableVertexAttribArray(2)
144
- glVertexAttribPointer(2, 3, GL_DOUBLE, GL_FALSE, 0, None)
145
-
146
- glDrawArrays(GL_TRIANGLES, 0, self.n_vertices[mat])
147
-
148
- glDisableVertexAttribArray(2)
149
- glDisableVertexAttribArray(1)
150
- glDisableVertexAttribArray(0)
151
-
152
- glBindBuffer(GL_ARRAY_BUFFER, 0)
153
-
154
- glUseProgram(0)
155
-
156
- glDisable(GL_MULTISAMPLE)
157
-
158
- self.draw_end()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl /normal_render.py DELETED
@@ -1,93 +0,0 @@
1
- '''
2
- MIT License
3
- Copyright (c) 2019 Shunsuke Saito, Zeng Huang, and Ryota Natsume
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
- The above copyright notice and this permission notice shall be included in all
11
- copies or substantial portions of the Software.
12
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
13
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
14
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
15
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
16
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
17
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
18
- SOFTWARE.
19
- '''
20
- import numpy as np
21
- import math
22
-
23
- from .framework import *
24
- from .norm_render import NormRender
25
-
26
-
27
- class NormalRender(NormRender):
28
- def __init__(self, width=1600, height=1200, name='Normal Renderer'):
29
- NormRender.__init__(self,
30
- width,
31
- height,
32
- name,
33
- program_files=['normal.vs', 'normal.fs'])
34
-
35
- self.norm_buffer = glGenBuffers(1)
36
-
37
- self.norm_data = None
38
-
39
- def set_normal_mesh(self, vertices, faces, norms, face_normals):
40
- NormRender.set_mesh(self, vertices, faces)
41
-
42
- self.norm_data = norms[face_normals.reshape([-1])]
43
-
44
- glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer)
45
- glBufferData(GL_ARRAY_BUFFER, self.norm_data, GL_STATIC_DRAW)
46
-
47
- glBindBuffer(GL_ARRAY_BUFFER, 0)
48
-
49
- def euler_to_rot_mat(self, r_x, r_y, r_z):
50
- R_x = np.array([[1, 0, 0], [0, math.cos(r_x), -math.sin(r_x)],
51
- [0, math.sin(r_x), math.cos(r_x)]])
52
-
53
- R_y = np.array([[math.cos(r_y), 0, math.sin(r_y)], [0, 1, 0],
54
- [-math.sin(r_y), 0, math.cos(r_y)]])
55
-
56
- R_z = np.array([[math.cos(r_z), -math.sin(r_z), 0],
57
- [math.sin(r_z), math.cos(r_z), 0], [0, 0, 1]])
58
-
59
- R = np.dot(R_z, np.dot(R_y, R_x))
60
-
61
- return R
62
-
63
- def draw(self):
64
- self.draw_init()
65
-
66
- glUseProgram(self.program)
67
- glUniformMatrix4fv(self.model_mat_unif, 1, GL_FALSE,
68
- self.model_view_matrix.transpose())
69
- glUniformMatrix4fv(self.persp_mat_unif, 1, GL_FALSE,
70
- self.projection_matrix.transpose())
71
-
72
- # Handle vertex buffer
73
- glBindBuffer(GL_ARRAY_BUFFER, self.vertex_buffer)
74
-
75
- glEnableVertexAttribArray(0)
76
- glVertexAttribPointer(0, self.vertex_dim, GL_DOUBLE, GL_FALSE, 0, None)
77
-
78
- # Handle normal buffer
79
- glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer)
80
-
81
- glEnableVertexAttribArray(1)
82
- glVertexAttribPointer(1, 3, GL_DOUBLE, GL_FALSE, 0, None)
83
-
84
- glDrawArrays(GL_TRIANGLES, 0, self.n_vertices)
85
-
86
- glDisableVertexAttribArray(1)
87
- glDisableVertexAttribArray(0)
88
-
89
- glBindBuffer(GL_ARRAY_BUFFER, 0)
90
-
91
- glUseProgram(0)
92
-
93
- self.draw_end()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl /prt_render.py DELETED
@@ -1,450 +0,0 @@
1
-
2
- # -*- coding: utf-8 -*-
3
-
4
- # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
5
- # holder of all proprietary rights on this computer program.
6
- # You can only use this computer program if you have closed
7
- # a license agreement with MPG or you get the right to use the computer
8
- # program from someone who is authorized to grant you that right.
9
- # Any use of the computer program without a valid license is prohibited and
10
- # liable to prosecution.
11
- #
12
- # Copyright©2019 Max-Planck-Gesellschaft zur Förderung
13
- # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
14
- # for Intelligent Systems. All rights reserved.
15
- #
16
- # Contact: [email protected]
17
-
18
- import numpy as np
19
- import random
20
-
21
- from .framework import *
22
- from .cam_render import CamRender
23
-
24
-
25
- class PRTRender(CamRender):
26
- def __init__(self,
27
- width=1600,
28
- height=1200,
29
- name='PRT Renderer',
30
- uv_mode=False,
31
- ms_rate=1,
32
- egl=False):
33
- program_files = ['prt.vs', 'prt.fs'
34
- ] if not uv_mode else ['prt_uv.vs', 'prt_uv.fs']
35
- CamRender.__init__(self,
36
- width,
37
- height,
38
- name,
39
- program_files=program_files,
40
- color_size=8,
41
- ms_rate=ms_rate,
42
- egl=egl)
43
-
44
- # WARNING: this differs from vertex_buffer and vertex_data in Render
45
- self.vert_buffer = {}
46
- self.vert_data = {}
47
-
48
- self.vert_label_buffer = {}
49
- self.vert_label_data = {}
50
-
51
- self.norm_buffer = {}
52
- self.norm_data = {}
53
-
54
- self.tan_buffer = {}
55
- self.tan_data = {}
56
-
57
- self.btan_buffer = {}
58
- self.btan_data = {}
59
-
60
- self.prt1_buffer = {}
61
- self.prt1_data = {}
62
-
63
- self.prt2_buffer = {}
64
- self.prt2_data = {}
65
-
66
- self.prt3_buffer = {}
67
- self.prt3_data = {}
68
-
69
- self.uv_buffer = {}
70
- self.uv_data = {}
71
-
72
- self.render_texture_mat = {}
73
-
74
- self.vertex_dim = {}
75
- self.n_vertices = {}
76
- self.label_dim = {}
77
-
78
- self.norm_mat_unif = glGetUniformLocation(self.program, 'NormMat')
79
- self.normalize_matrix = np.eye(4)
80
-
81
- self.shcoeff_unif = glGetUniformLocation(self.program, 'SHCoeffs')
82
- self.shcoeffs = np.zeros((9, 3))
83
- self.shcoeffs[0, :] = 1.0
84
- #self.shcoeffs[1:,:] = np.random.rand(8,3)
85
-
86
- self.hasAlbedoUnif = glGetUniformLocation(self.program, 'hasAlbedoMap')
87
- self.hasNormalUnif = glGetUniformLocation(self.program, 'hasNormalMap')
88
-
89
- self.analyticUnif = glGetUniformLocation(self.program, 'analytic')
90
- self.analytic = False
91
-
92
- self.rot_mat_unif = glGetUniformLocation(self.program, 'RotMat')
93
- self.rot_matrix = np.eye(3)
94
-
95
- def set_texture(self, mat_name, smplr_name, texture):
96
- # texture_image: H x W x 3
97
- width = texture.shape[1]
98
- height = texture.shape[0]
99
- texture = np.flip(texture, 0)
100
- img_data = np.fromstring(texture.tostring(), np.uint8)
101
-
102
- if mat_name not in self.render_texture_mat:
103
- self.render_texture_mat[mat_name] = {}
104
- if smplr_name in self.render_texture_mat[mat_name].keys():
105
- glDeleteTextures([self.render_texture_mat[mat_name][smplr_name]])
106
- del self.render_texture_mat[mat_name][smplr_name]
107
-
108
- self.render_texture_mat[mat_name][smplr_name] = glGenTextures(1)
109
- glActiveTexture(GL_TEXTURE0)
110
-
111
- glPixelStorei(GL_UNPACK_ALIGNMENT, 1)
112
- glBindTexture(GL_TEXTURE_2D,
113
- self.render_texture_mat[mat_name][smplr_name])
114
-
115
- glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_RGB,
116
- GL_UNSIGNED_BYTE, img_data)
117
-
118
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAX_LEVEL, 3)
119
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
120
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
121
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
122
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
123
- GL_LINEAR_MIPMAP_LINEAR)
124
-
125
- glGenerateMipmap(GL_TEXTURE_2D)
126
-
127
- def set_albedo(self, texture_image, mat_name='all'):
128
- self.set_texture(mat_name, 'AlbedoMap', texture_image)
129
-
130
- def set_normal_map(self, texture_image, mat_name='all'):
131
- self.set_texture(mat_name, 'NormalMap', texture_image)
132
-
133
- def set_mesh(self,
134
- vertices,
135
- faces,
136
- norms,
137
- faces_nml,
138
- uvs,
139
- faces_uvs,
140
- prt,
141
- faces_prt,
142
- tans,
143
- bitans,
144
- verts_label=None,
145
- mat_name='all'):
146
-
147
- self.vert_data[mat_name] = vertices[faces.reshape([-1])]
148
- self.vert_label_data[mat_name] = verts_label[faces.reshape([-1])]
149
- self.n_vertices[mat_name] = self.vert_data[mat_name].shape[0]
150
- self.vertex_dim[mat_name] = self.vert_data[mat_name].shape[1]
151
- self.label_dim[mat_name] = self.vert_label_data[mat_name].shape[1]
152
-
153
- if mat_name not in self.vert_buffer.keys():
154
- self.vert_buffer[mat_name] = glGenBuffers(1)
155
- glBindBuffer(GL_ARRAY_BUFFER, self.vert_buffer[mat_name])
156
- glBufferData(GL_ARRAY_BUFFER, self.vert_data[mat_name], GL_STATIC_DRAW)
157
-
158
- if mat_name not in self.vert_label_buffer.keys():
159
- self.vert_label_buffer[mat_name] = glGenBuffers(1)
160
- glBindBuffer(GL_ARRAY_BUFFER, self.vert_label_buffer[mat_name])
161
- glBufferData(GL_ARRAY_BUFFER, self.vert_label_data[mat_name],
162
- GL_STATIC_DRAW)
163
-
164
- self.uv_data[mat_name] = uvs[faces_uvs.reshape([-1])]
165
- if mat_name not in self.uv_buffer.keys():
166
- self.uv_buffer[mat_name] = glGenBuffers(1)
167
- glBindBuffer(GL_ARRAY_BUFFER, self.uv_buffer[mat_name])
168
- glBufferData(GL_ARRAY_BUFFER, self.uv_data[mat_name], GL_STATIC_DRAW)
169
-
170
- self.norm_data[mat_name] = norms[faces_nml.reshape([-1])]
171
- if mat_name not in self.norm_buffer.keys():
172
- self.norm_buffer[mat_name] = glGenBuffers(1)
173
- glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer[mat_name])
174
- glBufferData(GL_ARRAY_BUFFER, self.norm_data[mat_name], GL_STATIC_DRAW)
175
-
176
- self.tan_data[mat_name] = tans[faces_nml.reshape([-1])]
177
- if mat_name not in self.tan_buffer.keys():
178
- self.tan_buffer[mat_name] = glGenBuffers(1)
179
- glBindBuffer(GL_ARRAY_BUFFER, self.tan_buffer[mat_name])
180
- glBufferData(GL_ARRAY_BUFFER, self.tan_data[mat_name], GL_STATIC_DRAW)
181
-
182
- self.btan_data[mat_name] = bitans[faces_nml.reshape([-1])]
183
- if mat_name not in self.btan_buffer.keys():
184
- self.btan_buffer[mat_name] = glGenBuffers(1)
185
- glBindBuffer(GL_ARRAY_BUFFER, self.btan_buffer[mat_name])
186
- glBufferData(GL_ARRAY_BUFFER, self.btan_data[mat_name], GL_STATIC_DRAW)
187
-
188
- self.prt1_data[mat_name] = prt[faces_prt.reshape([-1])][:, :3]
189
- self.prt2_data[mat_name] = prt[faces_prt.reshape([-1])][:, 3:6]
190
- self.prt3_data[mat_name] = prt[faces_prt.reshape([-1])][:, 6:]
191
-
192
- if mat_name not in self.prt1_buffer.keys():
193
- self.prt1_buffer[mat_name] = glGenBuffers(1)
194
- if mat_name not in self.prt2_buffer.keys():
195
- self.prt2_buffer[mat_name] = glGenBuffers(1)
196
- if mat_name not in self.prt3_buffer.keys():
197
- self.prt3_buffer[mat_name] = glGenBuffers(1)
198
- glBindBuffer(GL_ARRAY_BUFFER, self.prt1_buffer[mat_name])
199
- glBufferData(GL_ARRAY_BUFFER, self.prt1_data[mat_name], GL_STATIC_DRAW)
200
- glBindBuffer(GL_ARRAY_BUFFER, self.prt2_buffer[mat_name])
201
- glBufferData(GL_ARRAY_BUFFER, self.prt2_data[mat_name], GL_STATIC_DRAW)
202
- glBindBuffer(GL_ARRAY_BUFFER, self.prt3_buffer[mat_name])
203
- glBufferData(GL_ARRAY_BUFFER, self.prt3_data[mat_name], GL_STATIC_DRAW)
204
-
205
- glBindBuffer(GL_ARRAY_BUFFER, 0)
206
-
207
- def set_mesh_mtl(self,
208
- vertices,
209
- faces,
210
- norms,
211
- faces_nml,
212
- uvs,
213
- faces_uvs,
214
- tans,
215
- bitans,
216
- prt,
217
- verts_label=None):
218
- for key in faces:
219
- self.vert_data[key] = vertices[faces[key].reshape([-1])]
220
- self.vert_label_data[key] = verts_label[faces[key].reshape([-1])]
221
- self.n_vertices[key] = self.vert_data[key].shape[0]
222
- self.vertex_dim[key] = self.vert_data[key].shape[1]
223
- self.label_dim[key] = self.vert_label_data[key].shape[1]
224
-
225
- if key not in self.vert_buffer.keys():
226
- self.vert_buffer[key] = glGenBuffers(1)
227
- glBindBuffer(GL_ARRAY_BUFFER, self.vert_buffer[key])
228
- glBufferData(GL_ARRAY_BUFFER, self.vert_data[key], GL_STATIC_DRAW)
229
-
230
- if key not in self.vert_label_buffer.keys():
231
- self.vert_label_buffer[key] = glGenBuffers(1)
232
- glBindBuffer(GL_ARRAY_BUFFER, self.vert_label_buffer[key])
233
- glBufferData(GL_ARRAY_BUFFER, self.vert_label_data[key],
234
- GL_STATIC_DRAW)
235
-
236
- self.uv_data[key] = uvs[faces_uvs[key].reshape([-1])]
237
- if key not in self.uv_buffer.keys():
238
- self.uv_buffer[key] = glGenBuffers(1)
239
- glBindBuffer(GL_ARRAY_BUFFER, self.uv_buffer[key])
240
- glBufferData(GL_ARRAY_BUFFER, self.uv_data[key], GL_STATIC_DRAW)
241
-
242
- self.norm_data[key] = norms[faces_nml[key].reshape([-1])]
243
- if key not in self.norm_buffer.keys():
244
- self.norm_buffer[key] = glGenBuffers(1)
245
- glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer[key])
246
- glBufferData(GL_ARRAY_BUFFER, self.norm_data[key], GL_STATIC_DRAW)
247
-
248
- self.tan_data[key] = tans[faces_nml[key].reshape([-1])]
249
- if key not in self.tan_buffer.keys():
250
- self.tan_buffer[key] = glGenBuffers(1)
251
- glBindBuffer(GL_ARRAY_BUFFER, self.tan_buffer[key])
252
- glBufferData(GL_ARRAY_BUFFER, self.tan_data[key], GL_STATIC_DRAW)
253
-
254
- self.btan_data[key] = bitans[faces_nml[key].reshape([-1])]
255
- if key not in self.btan_buffer.keys():
256
- self.btan_buffer[key] = glGenBuffers(1)
257
- glBindBuffer(GL_ARRAY_BUFFER, self.btan_buffer[key])
258
- glBufferData(GL_ARRAY_BUFFER, self.btan_data[key], GL_STATIC_DRAW)
259
-
260
- self.prt1_data[key] = prt[faces[key].reshape([-1])][:, :3]
261
- self.prt2_data[key] = prt[faces[key].reshape([-1])][:, 3:6]
262
- self.prt3_data[key] = prt[faces[key].reshape([-1])][:, 6:]
263
-
264
- if key not in self.prt1_buffer.keys():
265
- self.prt1_buffer[key] = glGenBuffers(1)
266
- if key not in self.prt2_buffer.keys():
267
- self.prt2_buffer[key] = glGenBuffers(1)
268
- if key not in self.prt3_buffer.keys():
269
- self.prt3_buffer[key] = glGenBuffers(1)
270
- glBindBuffer(GL_ARRAY_BUFFER, self.prt1_buffer[key])
271
- glBufferData(GL_ARRAY_BUFFER, self.prt1_data[key], GL_STATIC_DRAW)
272
- glBindBuffer(GL_ARRAY_BUFFER, self.prt2_buffer[key])
273
- glBufferData(GL_ARRAY_BUFFER, self.prt2_data[key], GL_STATIC_DRAW)
274
- glBindBuffer(GL_ARRAY_BUFFER, self.prt3_buffer[key])
275
- glBufferData(GL_ARRAY_BUFFER, self.prt3_data[key], GL_STATIC_DRAW)
276
-
277
- glBindBuffer(GL_ARRAY_BUFFER, 0)
278
-
279
- def cleanup(self):
280
-
281
- glBindBuffer(GL_ARRAY_BUFFER, 0)
282
- for key in self.vert_data:
283
- glDeleteBuffers(1, [self.vert_buffer[key]])
284
- glDeleteBuffers(1, [self.norm_buffer[key]])
285
- glDeleteBuffers(1, [self.uv_buffer[key]])
286
- glDeleteBuffers(1, [self.vert_label_buffer[key]])
287
-
288
- glDeleteBuffers(1, [self.tan_buffer[key]])
289
- glDeleteBuffers(1, [self.btan_buffer[key]])
290
- glDeleteBuffers(1, [self.prt1_buffer[key]])
291
- glDeleteBuffers(1, [self.prt2_buffer[key]])
292
- glDeleteBuffers(1, [self.prt3_buffer[key]])
293
-
294
- glDeleteBuffers(1, [])
295
-
296
- for smplr in self.render_texture_mat[key]:
297
- glDeleteTextures([self.render_texture_mat[key][smplr]])
298
-
299
- self.vert_buffer = {}
300
- self.vert_data = {}
301
-
302
- self.vert_label_buffer = {}
303
- self.vert_label_data = {}
304
-
305
- self.norm_buffer = {}
306
- self.norm_data = {}
307
-
308
- self.tan_buffer = {}
309
- self.tan_data = {}
310
-
311
- self.btan_buffer = {}
312
- self.btan_data = {}
313
-
314
- self.prt1_buffer = {}
315
- self.prt1_data = {}
316
-
317
- self.prt2_buffer = {}
318
- self.prt2_data = {}
319
-
320
- self.prt3_buffer = {}
321
- self.prt3_data = {}
322
-
323
- self.uv_buffer = {}
324
- self.uv_data = {}
325
-
326
- self.render_texture_mat = {}
327
-
328
- self.vertex_dim = {}
329
- self.n_vertices = {}
330
- self.label_dim = {}
331
-
332
- def randomize_sh(self):
333
- self.shcoeffs[0, :] = 0.8
334
- self.shcoeffs[1:, :] = 1.0 * np.random.rand(8, 3)
335
-
336
- def set_sh(self, sh):
337
- self.shcoeffs = sh
338
-
339
- def set_norm_mat(self, scale, center):
340
- N = np.eye(4)
341
- N[:3, :3] = scale * np.eye(3)
342
- N[:3, 3] = -scale * center
343
-
344
- self.normalize_matrix = N
345
-
346
- def draw(self):
347
- self.draw_init()
348
-
349
- glDisable(GL_BLEND)
350
- #glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
351
- glEnable(GL_MULTISAMPLE)
352
-
353
- glUseProgram(self.program)
354
- glUniformMatrix4fv(self.norm_mat_unif, 1, GL_FALSE,
355
- self.normalize_matrix.transpose())
356
- glUniformMatrix4fv(self.model_mat_unif, 1, GL_FALSE,
357
- self.model_view_matrix.transpose())
358
- glUniformMatrix4fv(self.persp_mat_unif, 1, GL_FALSE,
359
- self.projection_matrix.transpose())
360
-
361
- if 'AlbedoMap' in self.render_texture_mat['all']:
362
- glUniform1ui(self.hasAlbedoUnif, GLuint(1))
363
- else:
364
- glUniform1ui(self.hasAlbedoUnif, GLuint(0))
365
-
366
- if 'NormalMap' in self.render_texture_mat['all']:
367
- glUniform1ui(self.hasNormalUnif, GLuint(1))
368
- else:
369
- glUniform1ui(self.hasNormalUnif, GLuint(0))
370
-
371
- glUniform1ui(self.analyticUnif,
372
- GLuint(1) if self.analytic else GLuint(0))
373
-
374
- glUniform3fv(self.shcoeff_unif, 9, self.shcoeffs)
375
-
376
- glUniformMatrix3fv(self.rot_mat_unif, 1, GL_FALSE,
377
- self.rot_matrix.transpose())
378
-
379
- for mat in self.vert_buffer:
380
- # Handle vertex buffer
381
- glBindBuffer(GL_ARRAY_BUFFER, self.vert_buffer[mat])
382
- glEnableVertexAttribArray(0)
383
- glVertexAttribPointer(0, self.vertex_dim[mat], GL_DOUBLE, GL_FALSE,
384
- 0, None)
385
-
386
- # Handle normal buffer
387
- glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer[mat])
388
- glEnableVertexAttribArray(1)
389
- glVertexAttribPointer(1, 3, GL_DOUBLE, GL_FALSE, 0, None)
390
-
391
- # Handle uv buffer
392
- glBindBuffer(GL_ARRAY_BUFFER, self.uv_buffer[mat])
393
- glEnableVertexAttribArray(2)
394
- glVertexAttribPointer(2, 2, GL_DOUBLE, GL_FALSE, 0, None)
395
-
396
- # Handle tan buffer
397
- glBindBuffer(GL_ARRAY_BUFFER, self.tan_buffer[mat])
398
- glEnableVertexAttribArray(3)
399
- glVertexAttribPointer(3, 3, GL_DOUBLE, GL_FALSE, 0, None)
400
-
401
- # Handle btan buffer
402
- glBindBuffer(GL_ARRAY_BUFFER, self.btan_buffer[mat])
403
- glEnableVertexAttribArray(4)
404
- glVertexAttribPointer(4, 3, GL_DOUBLE, GL_FALSE, 0, None)
405
-
406
- # Handle PTR buffer
407
- glBindBuffer(GL_ARRAY_BUFFER, self.prt1_buffer[mat])
408
- glEnableVertexAttribArray(5)
409
- glVertexAttribPointer(5, 3, GL_DOUBLE, GL_FALSE, 0, None)
410
-
411
- glBindBuffer(GL_ARRAY_BUFFER, self.prt2_buffer[mat])
412
- glEnableVertexAttribArray(6)
413
- glVertexAttribPointer(6, 3, GL_DOUBLE, GL_FALSE, 0, None)
414
-
415
- glBindBuffer(GL_ARRAY_BUFFER, self.prt3_buffer[mat])
416
- glEnableVertexAttribArray(7)
417
- glVertexAttribPointer(7, 3, GL_DOUBLE, GL_FALSE, 0, None)
418
-
419
- # Handle vertex label buffer
420
- glBindBuffer(GL_ARRAY_BUFFER, self.vert_label_buffer[mat])
421
- glEnableVertexAttribArray(8)
422
- glVertexAttribPointer(8, self.label_dim[mat], GL_DOUBLE, GL_FALSE,
423
- 0, None)
424
-
425
- for i, smplr in enumerate(self.render_texture_mat[mat]):
426
- glActiveTexture(GL_TEXTURE0 + i)
427
- glBindTexture(GL_TEXTURE_2D,
428
- self.render_texture_mat[mat][smplr])
429
- glUniform1i(glGetUniformLocation(self.program, smplr), i)
430
-
431
- glDrawArrays(GL_TRIANGLES, 0, self.n_vertices[mat])
432
-
433
- glDisableVertexAttribArray(8)
434
- glDisableVertexAttribArray(7)
435
- glDisableVertexAttribArray(6)
436
- glDisableVertexAttribArray(5)
437
- glDisableVertexAttribArray(4)
438
- glDisableVertexAttribArray(3)
439
- glDisableVertexAttribArray(2)
440
- glDisableVertexAttribArray(1)
441
- glDisableVertexAttribArray(0)
442
-
443
- glBindBuffer(GL_ARRAY_BUFFER, 0)
444
-
445
- glUseProgram(0)
446
-
447
- glDisable(GL_BLEND)
448
- glDisable(GL_MULTISAMPLE)
449
-
450
- self.draw_end()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer / gl /render2.py DELETED
@@ -1,384 +0,0 @@
1
- '''
2
- MIT License
3
- Copyright (c) 2019 Shunsuke Saito, Zeng Huang, and Ryota Natsume
4
- Permission is hereby granted, free of charge, to any person obtaining a copy
5
- of this software and associated documentation files (the "Software"), to deal
6
- in the Software without restriction, including without limitation the rights
7
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
- copies of the Software, and to permit persons to whom the Software is
9
- furnished to do so, subject to the following conditions:
10
- The above copyright notice and this permission notice shall be included in all
11
- copies or substantial portions of the Software.
12
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
13
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
14
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
15
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
16
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
17
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
18
- SOFTWARE.
19
- '''
20
- import numpy as np
21
- from OpenGL.GLUT import *
22
- from .framework import *
23
-
24
- _glut_window = None
25
-
26
-
27
- class Render:
28
- def __init__(self,
29
- width=1600,
30
- height=1200,
31
- name='GL Renderer',
32
- program_files=['simple.fs', 'simple.vs'],
33
- color_size=1,
34
- ms_rate=1):
35
- self.width = width
36
- self.height = height
37
- self.name = name
38
- self.display_mode = GLUT_DOUBLE | GLUT_RGB | GLUT_DEPTH
39
- self.use_inverse_depth = False
40
-
41
- global _glut_window
42
- if _glut_window is None:
43
- glutInit()
44
- glutInitDisplayMode(self.display_mode)
45
- glutInitWindowSize(self.width, self.height)
46
- glutInitWindowPosition(0, 0)
47
- _glut_window = glutCreateWindow("My Render.")
48
-
49
- # glEnable(GL_DEPTH_CLAMP)
50
- glEnable(GL_DEPTH_TEST)
51
-
52
- glClampColor(GL_CLAMP_READ_COLOR, GL_FALSE)
53
- glClampColor(GL_CLAMP_FRAGMENT_COLOR, GL_FALSE)
54
- glClampColor(GL_CLAMP_VERTEX_COLOR, GL_FALSE)
55
-
56
- # init program
57
- shader_list = []
58
-
59
- for program_file in program_files:
60
- _, ext = os.path.splitext(program_file)
61
- if ext == '.vs':
62
- shader_list.append(loadShader(GL_VERTEX_SHADER, program_file))
63
- elif ext == '.fs':
64
- shader_list.append(loadShader(GL_FRAGMENT_SHADER,
65
- program_file))
66
- elif ext == '.gs':
67
- shader_list.append(loadShader(GL_GEOMETRY_SHADER,
68
- program_file))
69
-
70
- self.program = createProgram(shader_list)
71
-
72
- for shader in shader_list:
73
- glDeleteShader(shader)
74
-
75
- # Init uniform variables
76
- self.model_mat_unif = glGetUniformLocation(self.program, 'ModelMat')
77
- self.persp_mat_unif = glGetUniformLocation(self.program, 'PerspMat')
78
-
79
- self.vertex_buffer = glGenBuffers(1)
80
-
81
- # Init screen quad program and buffer
82
- self.quad_program, self.quad_buffer = self.init_quad_program()
83
-
84
- # Configure frame buffer
85
- self.frame_buffer = glGenFramebuffers(1)
86
- glBindFramebuffer(GL_FRAMEBUFFER, self.frame_buffer)
87
-
88
- self.intermediate_fbo = None
89
- if ms_rate > 1:
90
- # Configure texture buffer to render to
91
- self.color_buffer = []
92
- for i in range(color_size):
93
- color_buffer = glGenTextures(1)
94
- multi_sample_rate = ms_rate
95
- glBindTexture(GL_TEXTURE_2D_MULTISAMPLE, color_buffer)
96
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,
97
- GL_CLAMP_TO_EDGE)
98
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,
99
- GL_CLAMP_TO_EDGE)
100
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
101
- GL_LINEAR)
102
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
103
- GL_LINEAR)
104
- glTexImage2DMultisample(GL_TEXTURE_2D_MULTISAMPLE,
105
- multi_sample_rate, GL_RGBA32F,
106
- self.width, self.height, GL_TRUE)
107
- glBindTexture(GL_TEXTURE_2D_MULTISAMPLE, 0)
108
- glFramebufferTexture2D(GL_FRAMEBUFFER,
109
- GL_COLOR_ATTACHMENT0 + i,
110
- GL_TEXTURE_2D_MULTISAMPLE, color_buffer,
111
- 0)
112
- self.color_buffer.append(color_buffer)
113
-
114
- self.render_buffer = glGenRenderbuffers(1)
115
- glBindRenderbuffer(GL_RENDERBUFFER, self.render_buffer)
116
- glRenderbufferStorageMultisample(GL_RENDERBUFFER,
117
- multi_sample_rate,
118
- GL_DEPTH24_STENCIL8, self.width,
119
- self.height)
120
- glBindRenderbuffer(GL_RENDERBUFFER, 0)
121
- glFramebufferRenderbuffer(GL_FRAMEBUFFER,
122
- GL_DEPTH_STENCIL_ATTACHMENT,
123
- GL_RENDERBUFFER, self.render_buffer)
124
-
125
- attachments = []
126
- for i in range(color_size):
127
- attachments.append(GL_COLOR_ATTACHMENT0 + i)
128
- glDrawBuffers(color_size, attachments)
129
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
130
-
131
- self.intermediate_fbo = glGenFramebuffers(1)
132
- glBindFramebuffer(GL_FRAMEBUFFER, self.intermediate_fbo)
133
-
134
- self.screen_texture = []
135
- for i in range(color_size):
136
- screen_texture = glGenTextures(1)
137
- glBindTexture(GL_TEXTURE_2D, screen_texture)
138
- glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, self.width,
139
- self.height, 0, GL_RGBA, GL_FLOAT, None)
140
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
141
- GL_LINEAR)
142
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
143
- GL_LINEAR)
144
- glFramebufferTexture2D(GL_FRAMEBUFFER,
145
- GL_COLOR_ATTACHMENT0 + i, GL_TEXTURE_2D,
146
- screen_texture, 0)
147
- self.screen_texture.append(screen_texture)
148
-
149
- glDrawBuffers(color_size, attachments)
150
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
151
- else:
152
- self.color_buffer = []
153
- for i in range(color_size):
154
- color_buffer = glGenTextures(1)
155
- glBindTexture(GL_TEXTURE_2D, color_buffer)
156
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,
157
- GL_CLAMP_TO_EDGE)
158
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,
159
- GL_CLAMP_TO_EDGE)
160
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
161
- GL_NEAREST)
162
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
163
- GL_NEAREST)
164
- glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, self.width,
165
- self.height, 0, GL_RGBA, GL_FLOAT, None)
166
- glFramebufferTexture2D(GL_FRAMEBUFFER,
167
- GL_COLOR_ATTACHMENT0 + i, GL_TEXTURE_2D,
168
- color_buffer, 0)
169
- self.color_buffer.append(color_buffer)
170
-
171
- # Configure depth texture map to render to
172
- self.depth_buffer = glGenTextures(1)
173
- glBindTexture(GL_TEXTURE_2D, self.depth_buffer)
174
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
175
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)
176
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
177
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
178
- glTexParameteri(GL_TEXTURE_2D, GL_DEPTH_TEXTURE_MODE, GL_INTENSITY)
179
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_MODE,
180
- GL_COMPARE_R_TO_TEXTURE)
181
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FUNC, GL_LEQUAL)
182
- glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT, self.width,
183
- self.height, 0, GL_DEPTH_COMPONENT, GL_FLOAT, None)
184
- glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT,
185
- GL_TEXTURE_2D, self.depth_buffer, 0)
186
-
187
- attachments = []
188
- for i in range(color_size):
189
- attachments.append(GL_COLOR_ATTACHMENT0 + i)
190
- glDrawBuffers(color_size, attachments)
191
- self.screen_texture = self.color_buffer
192
-
193
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
194
-
195
- # Configure texture buffer if needed
196
- self.render_texture = None
197
-
198
- # NOTE: original render_texture only support one input
199
- # this is tentative member of this issue
200
- self.render_texture_v2 = {}
201
-
202
- # Inner storage for buffer data
203
- self.vertex_data = None
204
- self.vertex_dim = None
205
- self.n_vertices = None
206
-
207
- self.model_view_matrix = None
208
- self.projection_matrix = None
209
-
210
- glutDisplayFunc(self.display)
211
-
212
- def init_quad_program(self):
213
- shader_list = []
214
-
215
- shader_list.append(loadShader(GL_VERTEX_SHADER, "quad.vs"))
216
- shader_list.append(loadShader(GL_FRAGMENT_SHADER, "quad.fs"))
217
-
218
- the_program = createProgram(shader_list)
219
-
220
- for shader in shader_list:
221
- glDeleteShader(shader)
222
-
223
- # vertex attributes for a quad that fills the entire screen in Normalized Device Coordinates.
224
- # positions # texCoords
225
- quad_vertices = np.array([
226
- -1.0, 1.0, 0.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0, -1.0, 1.0, 0.0,
227
- -1.0, 1.0, 0.0, 1.0, 1.0, -1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0
228
- ])
229
-
230
- quad_buffer = glGenBuffers(1)
231
- glBindBuffer(GL_ARRAY_BUFFER, quad_buffer)
232
- glBufferData(GL_ARRAY_BUFFER, quad_vertices, GL_STATIC_DRAW)
233
-
234
- glBindBuffer(GL_ARRAY_BUFFER, 0)
235
-
236
- return the_program, quad_buffer
237
-
238
- def set_mesh(self, vertices, faces):
239
- self.vertex_data = vertices[faces.reshape([-1])]
240
- self.vertex_dim = self.vertex_data.shape[1]
241
- self.n_vertices = self.vertex_data.shape[0]
242
-
243
- glBindBuffer(GL_ARRAY_BUFFER, self.vertex_buffer)
244
- glBufferData(GL_ARRAY_BUFFER, self.vertex_data, GL_STATIC_DRAW)
245
-
246
- glBindBuffer(GL_ARRAY_BUFFER, 0)
247
-
248
- def set_viewpoint(self, projection, model_view):
249
- self.projection_matrix = projection
250
- self.model_view_matrix = model_view
251
-
252
- def draw_init(self):
253
- glBindFramebuffer(GL_FRAMEBUFFER, self.frame_buffer)
254
- glEnable(GL_DEPTH_TEST)
255
-
256
- # glClearColor(0.0, 0.0, 0.0, 0.0)
257
- glClearColor(1.0, 1.0, 1.0, 0.0) # Black background
258
-
259
- if self.use_inverse_depth:
260
- glDepthFunc(GL_GREATER)
261
- glClearDepth(0.0)
262
- else:
263
- glDepthFunc(GL_LESS)
264
- glClearDepth(1.0)
265
- glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
266
-
267
- def draw_end(self):
268
- if self.intermediate_fbo is not None:
269
- for i in range(len(self.color_buffer)):
270
- glBindFramebuffer(GL_READ_FRAMEBUFFER, self.frame_buffer)
271
- glReadBuffer(GL_COLOR_ATTACHMENT0 + i)
272
- glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self.intermediate_fbo)
273
- glDrawBuffer(GL_COLOR_ATTACHMENT0 + i)
274
- glBlitFramebuffer(0, 0, self.width, self.height, 0, 0,
275
- self.width, self.height, GL_COLOR_BUFFER_BIT,
276
- GL_NEAREST)
277
-
278
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
279
- glDepthFunc(GL_LESS)
280
- glClearDepth(1.0)
281
-
282
- def draw(self):
283
- self.draw_init()
284
-
285
- glUseProgram(self.program)
286
- glUniformMatrix4fv(self.model_mat_unif, 1, GL_FALSE,
287
- self.model_view_matrix.transpose())
288
- glUniformMatrix4fv(self.persp_mat_unif, 1, GL_FALSE,
289
- self.projection_matrix.transpose())
290
-
291
- glBindBuffer(GL_ARRAY_BUFFER, self.vertex_buffer)
292
-
293
- glEnableVertexAttribArray(0)
294
- glVertexAttribPointer(0, self.vertex_dim, GL_DOUBLE, GL_FALSE, 0, None)
295
-
296
- glDrawArrays(GL_TRIANGLES, 0, self.n_vertices)
297
-
298
- glDisableVertexAttribArray(0)
299
-
300
- glBindBuffer(GL_ARRAY_BUFFER, 0)
301
-
302
- glUseProgram(0)
303
-
304
- self.draw_end()
305
-
306
- def get_color(self, color_id=0):
307
- glBindFramebuffer(
308
- GL_FRAMEBUFFER, self.intermediate_fbo
309
- if self.intermediate_fbo is not None else self.frame_buffer)
310
- glReadBuffer(GL_COLOR_ATTACHMENT0 + color_id)
311
- data = glReadPixels(0,
312
- 0,
313
- self.width,
314
- self.height,
315
- GL_RGBA,
316
- GL_FLOAT,
317
- outputType=None)
318
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
319
- rgb = data.reshape(self.height, self.width, -1)
320
- rgb = np.flip(rgb, 0)
321
- return rgb
322
-
323
- def get_z_value(self):
324
- glBindFramebuffer(GL_FRAMEBUFFER, self.frame_buffer)
325
- data = glReadPixels(0,
326
- 0,
327
- self.width,
328
- self.height,
329
- GL_DEPTH_COMPONENT,
330
- GL_FLOAT,
331
- outputType=None)
332
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
333
- z = data.reshape(self.height, self.width)
334
- z = np.flip(z, 0)
335
- return z
336
-
337
- def display(self):
338
- # First we draw a scene.
339
- # Notice the result is stored in the texture buffer.
340
- self.draw()
341
-
342
- # Then we return to the default frame buffer since we will display on the screen.
343
- glBindFramebuffer(GL_FRAMEBUFFER, 0)
344
-
345
- # Do the clean-up.
346
- # glClearColor(0.0, 0.0, 0.0, 0.0) #Black background
347
- glClearColor(1.0, 1.0, 1.0, 0.0) # Black background
348
- glClear(GL_COLOR_BUFFER_BIT)
349
-
350
- # We draw a rectangle which covers the whole screen.
351
- glUseProgram(self.quad_program)
352
- glBindBuffer(GL_ARRAY_BUFFER, self.quad_buffer)
353
-
354
- size_of_double = 8
355
- glEnableVertexAttribArray(0)
356
- glVertexAttribPointer(0, 2, GL_DOUBLE, GL_FALSE, 4 * size_of_double,
357
- None)
358
- glEnableVertexAttribArray(1)
359
- glVertexAttribPointer(1, 2, GL_DOUBLE, GL_FALSE, 4 * size_of_double,
360
- c_void_p(2 * size_of_double))
361
-
362
- glDisable(GL_DEPTH_TEST)
363
-
364
- # The stored texture is then mapped to this rectangle.
365
- # properly assing color buffer texture
366
- glActiveTexture(GL_TEXTURE0)
367
- glBindTexture(GL_TEXTURE_2D, self.screen_texture[0])
368
- glUniform1i(glGetUniformLocation(self.quad_program, 'screenTexture'),
369
- 0)
370
-
371
- glDrawArrays(GL_TRIANGLES, 0, 6)
372
-
373
- glDisableVertexAttribArray(1)
374
- glDisableVertexAttribArray(0)
375
-
376
- glEnable(GL_DEPTH_TEST)
377
- glBindBuffer(GL_ARRAY_BUFFER, 0)
378
- glUseProgram(0)
379
-
380
- glutSwapBuffers()
381
- glutPostRedisplay()
382
-
383
- def show(self):
384
- glutMainLoop()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer /glm.py DELETED
@@ -1,143 +0,0 @@
1
-
2
- # -*- coding: utf-8 -*-
3
-
4
- # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
5
- # holder of all proprietary rights on this computer program.
6
- # You can only use this computer program if you have closed
7
- # a license agreement with MPG or you get the right to use the computer
8
- # program from someone who is authorized to grant you that right.
9
- # Any use of the computer program without a valid license is prohibited and
10
- # liable to prosecution.
11
- #
12
- # Copyright©2019 Max-Planck-Gesellschaft zur Förderung
13
- # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
14
- # for Intelligent Systems. All rights reserved.
15
- #
16
- # Contact: [email protected]
17
-
18
- import numpy as np
19
-
20
-
21
- def vec3(x, y, z):
22
- return np.array([x, y, z], dtype=np.float32)
23
-
24
-
25
- def radians(v):
26
- return np.radians(v)
27
-
28
-
29
- def identity():
30
- return np.identity(4, dtype=np.float32)
31
-
32
-
33
- def empty():
34
- return np.zeros([4, 4], dtype=np.float32)
35
-
36
-
37
- def magnitude(v):
38
- return np.linalg.norm(v)
39
-
40
-
41
- def normalize(v):
42
- m = magnitude(v)
43
- return v if m == 0 else v / m
44
-
45
-
46
- def dot(u, v):
47
- return np.sum(u * v)
48
-
49
-
50
- def cross(u, v):
51
- res = vec3(0, 0, 0)
52
- res[0] = u[1] * v[2] - u[2] * v[1]
53
- res[1] = u[2] * v[0] - u[0] * v[2]
54
- res[2] = u[0] * v[1] - u[1] * v[0]
55
- return res
56
-
57
-
58
- # below functions can be optimized
59
-
60
-
61
- def translate(m, v):
62
- res = np.copy(m)
63
- res[:, 3] = m[:, 0] * v[0] + m[:, 1] * v[1] + m[:, 2] * v[2] + m[:, 3]
64
- return res
65
-
66
-
67
- def rotate(m, angle, v):
68
- a = angle
69
- c = np.cos(a)
70
- s = np.sin(a)
71
-
72
- axis = normalize(v)
73
- temp = (1 - c) * axis
74
-
75
- rot = empty()
76
- rot[0][0] = c + temp[0] * axis[0]
77
- rot[0][1] = temp[0] * axis[1] + s * axis[2]
78
- rot[0][2] = temp[0] * axis[2] - s * axis[1]
79
-
80
- rot[1][0] = temp[1] * axis[0] - s * axis[2]
81
- rot[1][1] = c + temp[1] * axis[1]
82
- rot[1][2] = temp[1] * axis[2] + s * axis[0]
83
-
84
- rot[2][0] = temp[2] * axis[0] + s * axis[1]
85
- rot[2][1] = temp[2] * axis[1] - s * axis[0]
86
- rot[2][2] = c + temp[2] * axis[2]
87
-
88
- res = empty()
89
- res[:, 0] = m[:, 0] * rot[0][0] + m[:, 1] * rot[0][1] + m[:, 2] * rot[0][2]
90
- res[:, 1] = m[:, 0] * rot[1][0] + m[:, 1] * rot[1][1] + m[:, 2] * rot[1][2]
91
- res[:, 2] = m[:, 0] * rot[2][0] + m[:, 1] * rot[2][1] + m[:, 2] * rot[2][2]
92
- res[:, 3] = m[:, 3]
93
- return res
94
-
95
-
96
- def perspective(fovy, aspect, zNear, zFar):
97
- tanHalfFovy = np.tan(fovy / 2)
98
-
99
- res = empty()
100
- res[0][0] = 1 / (aspect * tanHalfFovy)
101
- res[1][1] = 1 / (tanHalfFovy)
102
- res[2][3] = -1
103
- res[2][2] = -(zFar + zNear) / (zFar - zNear)
104
- res[3][2] = -(2 * zFar * zNear) / (zFar - zNear)
105
-
106
- return res.T
107
-
108
-
109
- def ortho(left, right, bottom, top, zNear, zFar):
110
- # res = np.ones([4, 4], dtype=np.float32)
111
- res = identity()
112
- res[0][0] = 2 / (right - left)
113
- res[1][1] = 2 / (top - bottom)
114
- res[2][2] = -2 / (zFar - zNear)
115
- res[3][0] = -(right + left) / (right - left)
116
- res[3][1] = -(top + bottom) / (top - bottom)
117
- res[3][2] = -(zFar + zNear) / (zFar - zNear)
118
- return res.T
119
-
120
-
121
- def lookat(eye, center, up):
122
- f = normalize(center - eye)
123
- s = normalize(cross(f, up))
124
- u = cross(s, f)
125
-
126
- res = identity()
127
- res[0][0] = s[0]
128
- res[1][0] = s[1]
129
- res[2][0] = s[2]
130
- res[0][1] = u[0]
131
- res[1][1] = u[1]
132
- res[2][1] = u[2]
133
- res[0][2] = -f[0]
134
- res[1][2] = -f[1]
135
- res[2][2] = -f[2]
136
- res[3][0] = -dot(s, eye)
137
- res[3][1] = -dot(u, eye)
138
- res[3][2] = -dot(f, eye)
139
- return res.T
140
-
141
-
142
- def transform(d, m):
143
- return np.dot(m, d.T).T
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer /mesh.py DELETED
@@ -1,526 +0,0 @@
1
-
2
- # -*- coding: utf-8 -*-
3
-
4
- # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
5
- # holder of all proprietary rights on this computer program.
6
- # You can only use this computer program if you have closed
7
- # a license agreement with MPG or you get the right to use the computer
8
- # program from someone who is authorized to grant you that right.
9
- # Any use of the computer program without a valid license is prohibited and
10
- # liable to prosecution.
11
- #
12
- # Copyright©2019 Max-Planck-Gesellschaft zur Förderung
13
- # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
14
- # for Intelligent Systems. All rights reserved.
15
- #
16
- # Contact: [email protected]
17
-
18
- from lib.dataset.mesh_util import SMPLX
19
- from lib.common.render_utils import face_vertices
20
- import numpy as np
21
- import lib.smplx as smplx
22
- import trimesh
23
- import torch
24
- import torch.nn.functional as F
25
-
26
- model_init_params = dict(
27
- gender='male',
28
- model_type='smplx',
29
- model_path=SMPLX().model_dir,
30
- create_global_orient=False,
31
- create_body_pose=False,
32
- create_betas=False,
33
- create_left_hand_pose=False,
34
- create_right_hand_pose=False,
35
- create_expression=False,
36
- create_jaw_pose=False,
37
- create_leye_pose=False,
38
- create_reye_pose=False,
39
- create_transl=False,
40
- num_pca_comps=12)
41
-
42
-
43
- def get_smpl_model(model_type, gender): return smplx.create(
44
- **model_init_params)
45
-
46
-
47
- def normalization(data):
48
- _range = np.max(data) - np.min(data)
49
- return ((data - np.min(data)) / _range)
50
-
51
-
52
- def sigmoid(x):
53
- z = 1 / (1 + np.exp(-x))
54
- return z
55
-
56
-
57
- def load_fit_body(fitted_path, scale, smpl_type='smplx', smpl_gender='neutral', noise_dict=None):
58
-
59
- param = np.load(fitted_path, allow_pickle=True)
60
- for key in param.keys():
61
- param[key] = torch.as_tensor(param[key])
62
-
63
- smpl_model = get_smpl_model(smpl_type, smpl_gender)
64
- model_forward_params = dict(betas=param['betas'],
65
- global_orient=param['global_orient'],
66
- body_pose=param['body_pose'],
67
- left_hand_pose=param['left_hand_pose'],
68
- right_hand_pose=param['right_hand_pose'],
69
- jaw_pose=param['jaw_pose'],
70
- leye_pose=param['leye_pose'],
71
- reye_pose=param['reye_pose'],
72
- expression=param['expression'],
73
- return_verts=True)
74
-
75
- if noise_dict is not None:
76
- model_forward_params.update(noise_dict)
77
-
78
- smpl_out = smpl_model(**model_forward_params)
79
-
80
- smpl_verts = (
81
- (smpl_out.vertices[0] * param['scale'] + param['translation']) * scale).detach()
82
- smpl_joints = (
83
- (smpl_out.joints[0] * param['scale'] + param['translation']) * scale).detach()
84
- smpl_mesh = trimesh.Trimesh(smpl_verts,
85
- smpl_model.faces,
86
- process=False, maintain_order=True)
87
-
88
- return smpl_mesh, smpl_joints
89
-
90
-
91
- def load_ori_fit_body(fitted_path, smpl_type='smplx', smpl_gender='neutral'):
92
-
93
- param = np.load(fitted_path, allow_pickle=True)
94
- for key in param.keys():
95
- param[key] = torch.as_tensor(param[key])
96
-
97
- smpl_model = get_smpl_model(smpl_type, smpl_gender)
98
- model_forward_params = dict(betas=param['betas'],
99
- global_orient=param['global_orient'],
100
- body_pose=param['body_pose'],
101
- left_hand_pose=param['left_hand_pose'],
102
- right_hand_pose=param['right_hand_pose'],
103
- jaw_pose=param['jaw_pose'],
104
- leye_pose=param['leye_pose'],
105
- reye_pose=param['reye_pose'],
106
- expression=param['expression'],
107
- return_verts=True)
108
-
109
- smpl_out = smpl_model(**model_forward_params)
110
-
111
- smpl_verts = smpl_out.vertices[0].detach()
112
- smpl_mesh = trimesh.Trimesh(smpl_verts,
113
- smpl_model.faces,
114
- process=False, maintain_order=True)
115
-
116
- return smpl_mesh
117
-
118
-
119
- def save_obj_mesh(mesh_path, verts, faces):
120
- file = open(mesh_path, 'w')
121
- for v in verts:
122
- file.write('v %.4f %.4f %.4f\n' % (v[0], v[1], v[2]))
123
- for f in faces:
124
- f_plus = f + 1
125
- file.write('f %d %d %d\n' % (f_plus[0], f_plus[1], f_plus[2]))
126
- file.close()
127
-
128
-
129
- # https://github.com/ratcave/wavefront_reader
130
- def read_mtlfile(fname):
131
- materials = {}
132
- with open(fname) as f:
133
- lines = f.read().splitlines()
134
-
135
- for line in lines:
136
- if line:
137
- split_line = line.strip().split(' ', 1)
138
- if len(split_line) < 2:
139
- continue
140
-
141
- prefix, data = split_line[0], split_line[1]
142
- if 'newmtl' in prefix:
143
- material = {}
144
- materials[data] = material
145
- elif materials:
146
- if data:
147
- split_data = data.strip().split(' ')
148
-
149
- # assume texture maps are in the same level
150
- # WARNING: do not include space in your filename!!
151
- if 'map' in prefix:
152
- material[prefix] = split_data[-1].split('\\')[-1]
153
- elif len(split_data) > 1:
154
- material[prefix] = tuple(float(d) for d in split_data)
155
- else:
156
- try:
157
- material[prefix] = int(data)
158
- except ValueError:
159
- material[prefix] = float(data)
160
-
161
- return materials
162
-
163
-
164
- def load_obj_mesh_mtl(mesh_file):
165
- vertex_data = []
166
- norm_data = []
167
- uv_data = []
168
-
169
- face_data = []
170
- face_norm_data = []
171
- face_uv_data = []
172
-
173
- # face per material
174
- face_data_mat = {}
175
- face_norm_data_mat = {}
176
- face_uv_data_mat = {}
177
-
178
- # current material name
179
- mtl_data = None
180
- cur_mat = None
181
-
182
- if isinstance(mesh_file, str):
183
- f = open(mesh_file, "r")
184
- else:
185
- f = mesh_file
186
- for line in f:
187
- if isinstance(line, bytes):
188
- line = line.decode("utf-8")
189
- if line.startswith('#'):
190
- continue
191
- values = line.split()
192
- if not values:
193
- continue
194
-
195
- if values[0] == 'v':
196
- v = list(map(float, values[1:4]))
197
- vertex_data.append(v)
198
- elif values[0] == 'vn':
199
- vn = list(map(float, values[1:4]))
200
- norm_data.append(vn)
201
- elif values[0] == 'vt':
202
- vt = list(map(float, values[1:3]))
203
- uv_data.append(vt)
204
- elif values[0] == 'mtllib':
205
- mtl_data = read_mtlfile(
206
- mesh_file.replace(mesh_file.split('/')[-1], values[1]))
207
- elif values[0] == 'usemtl':
208
- cur_mat = values[1]
209
- elif values[0] == 'f':
210
- # local triangle data
211
- l_face_data = []
212
- l_face_uv_data = []
213
- l_face_norm_data = []
214
-
215
- # quad mesh
216
- if len(values) > 4:
217
- f = list(
218
- map(
219
- lambda x: int(x.split('/')[0]) if int(x.split('/')[0])
220
- < 0 else int(x.split('/')[0]) - 1, values[1:4]))
221
- l_face_data.append(f)
222
- f = list(
223
- map(
224
- lambda x: int(x.split('/')[0])
225
- if int(x.split('/')[0]) < 0 else int(x.split('/')[0]) -
226
- 1, [values[3], values[4], values[1]]))
227
- l_face_data.append(f)
228
- # tri mesh
229
- else:
230
- f = list(
231
- map(
232
- lambda x: int(x.split('/')[0]) if int(x.split('/')[0])
233
- < 0 else int(x.split('/')[0]) - 1, values[1:4]))
234
- l_face_data.append(f)
235
- # deal with texture
236
- if len(values[1].split('/')) >= 2:
237
- # quad mesh
238
- if len(values) > 4:
239
- f = list(
240
- map(
241
- lambda x: int(x.split('/')[1])
242
- if int(x.split('/')[1]) < 0 else int(
243
- x.split('/')[1]) - 1, values[1:4]))
244
- l_face_uv_data.append(f)
245
- f = list(
246
- map(
247
- lambda x: int(x.split('/')[1])
248
- if int(x.split('/')[1]) < 0 else int(
249
- x.split('/')[1]) - 1,
250
- [values[3], values[4], values[1]]))
251
- l_face_uv_data.append(f)
252
- # tri mesh
253
- elif len(values[1].split('/')[1]) != 0:
254
- f = list(
255
- map(
256
- lambda x: int(x.split('/')[1])
257
- if int(x.split('/')[1]) < 0 else int(
258
- x.split('/')[1]) - 1, values[1:4]))
259
- l_face_uv_data.append(f)
260
- # deal with normal
261
- if len(values[1].split('/')) == 3:
262
- # quad mesh
263
- if len(values) > 4:
264
- f = list(
265
- map(
266
- lambda x: int(x.split('/')[2])
267
- if int(x.split('/')[2]) < 0 else int(
268
- x.split('/')[2]) - 1, values[1:4]))
269
- l_face_norm_data.append(f)
270
- f = list(
271
- map(
272
- lambda x: int(x.split('/')[2])
273
- if int(x.split('/')[2]) < 0 else int(
274
- x.split('/')[2]) - 1,
275
- [values[3], values[4], values[1]]))
276
- l_face_norm_data.append(f)
277
- # tri mesh
278
- elif len(values[1].split('/')[2]) != 0:
279
- f = list(
280
- map(
281
- lambda x: int(x.split('/')[2])
282
- if int(x.split('/')[2]) < 0 else int(
283
- x.split('/')[2]) - 1, values[1:4]))
284
- l_face_norm_data.append(f)
285
-
286
- face_data += l_face_data
287
- face_uv_data += l_face_uv_data
288
- face_norm_data += l_face_norm_data
289
-
290
- if cur_mat is not None:
291
- if cur_mat not in face_data_mat.keys():
292
- face_data_mat[cur_mat] = []
293
- if cur_mat not in face_uv_data_mat.keys():
294
- face_uv_data_mat[cur_mat] = []
295
- if cur_mat not in face_norm_data_mat.keys():
296
- face_norm_data_mat[cur_mat] = []
297
- face_data_mat[cur_mat] += l_face_data
298
- face_uv_data_mat[cur_mat] += l_face_uv_data
299
- face_norm_data_mat[cur_mat] += l_face_norm_data
300
-
301
- vertices = np.array(vertex_data)
302
- faces = np.array(face_data)
303
-
304
- norms = np.array(norm_data)
305
- norms = normalize_v3(norms)
306
- face_normals = np.array(face_norm_data)
307
-
308
- uvs = np.array(uv_data)
309
- face_uvs = np.array(face_uv_data)
310
-
311
- out_tuple = (vertices, faces, norms, face_normals, uvs, face_uvs)
312
-
313
- if cur_mat is not None and mtl_data is not None:
314
- for key in face_data_mat:
315
- face_data_mat[key] = np.array(face_data_mat[key])
316
- face_uv_data_mat[key] = np.array(face_uv_data_mat[key])
317
- face_norm_data_mat[key] = np.array(face_norm_data_mat[key])
318
-
319
- out_tuple += (face_data_mat, face_norm_data_mat, face_uv_data_mat,
320
- mtl_data)
321
-
322
- return out_tuple
323
-
324
-
325
- def load_scan(mesh_file, with_normal=False, with_texture=False):
326
- vertex_data = []
327
- norm_data = []
328
- uv_data = []
329
-
330
- face_data = []
331
- face_norm_data = []
332
- face_uv_data = []
333
-
334
- if isinstance(mesh_file, str):
335
- f = open(mesh_file, "r")
336
- else:
337
- f = mesh_file
338
- for line in f:
339
- if isinstance(line, bytes):
340
- line = line.decode("utf-8")
341
- if line.startswith('#'):
342
- continue
343
- values = line.split()
344
- if not values:
345
- continue
346
-
347
- if values[0] == 'v':
348
- v = list(map(float, values[1:4]))
349
- vertex_data.append(v)
350
- elif values[0] == 'vn':
351
- vn = list(map(float, values[1:4]))
352
- norm_data.append(vn)
353
- elif values[0] == 'vt':
354
- vt = list(map(float, values[1:3]))
355
- uv_data.append(vt)
356
-
357
- elif values[0] == 'f':
358
- # quad mesh
359
- if len(values) > 4:
360
- f = list(map(lambda x: int(x.split('/')[0]), values[1:4]))
361
- face_data.append(f)
362
- f = list(
363
- map(lambda x: int(x.split('/')[0]),
364
- [values[3], values[4], values[1]]))
365
- face_data.append(f)
366
- # tri mesh
367
- else:
368
- f = list(map(lambda x: int(x.split('/')[0]), values[1:4]))
369
- face_data.append(f)
370
-
371
- # deal with texture
372
- if len(values[1].split('/')) >= 2:
373
- # quad mesh
374
- if len(values) > 4:
375
- f = list(map(lambda x: int(x.split('/')[1]), values[1:4]))
376
- face_uv_data.append(f)
377
- f = list(
378
- map(lambda x: int(x.split('/')[1]),
379
- [values[3], values[4], values[1]]))
380
- face_uv_data.append(f)
381
- # tri mesh
382
- elif len(values[1].split('/')[1]) != 0:
383
- f = list(map(lambda x: int(x.split('/')[1]), values[1:4]))
384
- face_uv_data.append(f)
385
- # deal with normal
386
- if len(values[1].split('/')) == 3:
387
- # quad mesh
388
- if len(values) > 4:
389
- f = list(map(lambda x: int(x.split('/')[2]), values[1:4]))
390
- face_norm_data.append(f)
391
- f = list(
392
- map(lambda x: int(x.split('/')[2]),
393
- [values[3], values[4], values[1]]))
394
- face_norm_data.append(f)
395
- # tri mesh
396
- elif len(values[1].split('/')[2]) != 0:
397
- f = list(map(lambda x: int(x.split('/')[2]), values[1:4]))
398
- face_norm_data.append(f)
399
-
400
- vertices = np.array(vertex_data)
401
- faces = np.array(face_data) - 1
402
-
403
- if with_texture and with_normal:
404
- uvs = np.array(uv_data)
405
- face_uvs = np.array(face_uv_data) - 1
406
- norms = np.array(norm_data)
407
- if norms.shape[0] == 0:
408
- norms = compute_normal(vertices, faces)
409
- face_normals = faces
410
- else:
411
- norms = normalize_v3(norms)
412
- face_normals = np.array(face_norm_data) - 1
413
- return vertices, faces, norms, face_normals, uvs, face_uvs
414
-
415
- if with_texture:
416
- uvs = np.array(uv_data)
417
- face_uvs = np.array(face_uv_data) - 1
418
- return vertices, faces, uvs, face_uvs
419
-
420
- if with_normal:
421
- norms = np.array(norm_data)
422
- norms = normalize_v3(norms)
423
- face_normals = np.array(face_norm_data) - 1
424
- return vertices, faces, norms, face_normals
425
-
426
- return vertices, faces
427
-
428
-
429
- def normalize_v3(arr):
430
- ''' Normalize a numpy array of 3 component vectors shape=(n,3) '''
431
- lens = np.sqrt(arr[:, 0]**2 + arr[:, 1]**2 + arr[:, 2]**2)
432
- eps = 0.00000001
433
- lens[lens < eps] = eps
434
- arr[:, 0] /= lens
435
- arr[:, 1] /= lens
436
- arr[:, 2] /= lens
437
- return arr
438
-
439
-
440
- def compute_normal(vertices, faces):
441
- # Create a zeroed array with the same type and shape as our vertices i.e., per vertex normal
442
- norm = np.zeros(vertices.shape, dtype=vertices.dtype)
443
- # Create an indexed view into the vertex array using the array of three indices for triangles
444
- tris = vertices[faces]
445
- # Calculate the normal for all the triangles, by taking the cross product of the vectors v1-v0, and v2-v0 in each triangle
446
- n = np.cross(tris[::, 1] - tris[::, 0], tris[::, 2] - tris[::, 0])
447
- # n is now an array of normals per triangle. The length of each normal is dependent the vertices,
448
- # we need to normalize these, so that our next step weights each normal equally.
449
- normalize_v3(n)
450
- # now we have a normalized array of normals, one per triangle, i.e., per triangle normals.
451
- # But instead of one per triangle (i.e., flat shading), we add to each vertex in that triangle,
452
- # the triangles' normal. Multiple triangles would then contribute to every vertex, so we need to normalize again afterwards.
453
- # The cool part, we can actually add the normals through an indexed view of our (zeroed) per vertex normal array
454
- norm[faces[:, 0]] += n
455
- norm[faces[:, 1]] += n
456
- norm[faces[:, 2]] += n
457
- normalize_v3(norm)
458
-
459
- return norm
460
-
461
-
462
- def compute_normal_batch(vertices, faces):
463
-
464
- bs, nv = vertices.shape[:2]
465
- bs, nf = faces.shape[:2]
466
-
467
- vert_norm = torch.zeros(bs * nv, 3).type_as(vertices)
468
- tris = face_vertices(vertices, faces)
469
- face_norm = F.normalize(torch.cross(tris[:, :, 1] - tris[:, :, 0],
470
- tris[:, :, 2] - tris[:, :, 0]),
471
- dim=-1)
472
-
473
- faces = (faces +
474
- (torch.arange(bs).type_as(faces) * nv)[:, None, None]).view(
475
- -1, 3)
476
-
477
- vert_norm[faces[:, 0]] += face_norm.view(-1, 3)
478
- vert_norm[faces[:, 1]] += face_norm.view(-1, 3)
479
- vert_norm[faces[:, 2]] += face_norm.view(-1, 3)
480
-
481
- vert_norm = F.normalize(vert_norm, dim=-1).view(bs, nv, 3)
482
-
483
- return vert_norm
484
-
485
-
486
- # compute tangent and bitangent
487
- def compute_tangent(vertices, faces, normals, uvs, faceuvs):
488
- # NOTE: this could be numerically unstable around [0,0,1]
489
- # but other current solutions are pretty freaky somehow
490
- c1 = np.cross(normals, np.array([0, 1, 0.0]))
491
- tan = c1
492
- normalize_v3(tan)
493
- btan = np.cross(normals, tan)
494
-
495
- # NOTE: traditional version is below
496
-
497
- # pts_tris = vertices[faces]
498
- # uv_tris = uvs[faceuvs]
499
-
500
- # W = np.stack([pts_tris[::, 1] - pts_tris[::, 0], pts_tris[::, 2] - pts_tris[::, 0]],2)
501
- # UV = np.stack([uv_tris[::, 1] - uv_tris[::, 0], uv_tris[::, 2] - uv_tris[::, 0]], 1)
502
-
503
- # for i in range(W.shape[0]):
504
- # W[i,::] = W[i,::].dot(np.linalg.inv(UV[i,::]))
505
-
506
- # tan = np.zeros(vertices.shape, dtype=vertices.dtype)
507
- # tan[faces[:,0]] += W[:,:,0]
508
- # tan[faces[:,1]] += W[:,:,0]
509
- # tan[faces[:,2]] += W[:,:,0]
510
-
511
- # btan = np.zeros(vertices.shape, dtype=vertices.dtype)
512
- # btan[faces[:,0]] += W[:,:,1]
513
- # btan[faces[:,1]] += W[:,:,1]
514
- # btan[faces[:,2]] += W[:,:,1]
515
-
516
- # normalize_v3(tan)
517
-
518
- # ndott = np.sum(normals*tan, 1, keepdims=True)
519
- # tan = tan - ndott * normals
520
-
521
- # normalize_v3(btan)
522
- # normalize_v3(tan)
523
-
524
- # tan[np.sum(np.cross(normals, tan) * btan, 1) < 0,:] *= -1.0
525
-
526
- return tan, btan
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer /opengl_util.py DELETED
@@ -1,369 +0,0 @@
1
-
2
- # -*- coding: utf-8 -*-
3
-
4
- # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
5
- # holder of all proprietary rights on this computer program.
6
- # You can only use this computer program if you have closed
7
- # a license agreement with MPG or you get the right to use the computer
8
- # program from someone who is authorized to grant you that right.
9
- # Any use of the computer program without a valid license is prohibited and
10
- # liable to prosecution.
11
- #
12
- # Copyright©2019 Max-Planck-Gesellschaft zur Förderung
13
- # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
14
- # for Intelligent Systems. All rights reserved.
15
- #
16
- # Contact: [email protected]
17
-
18
- import os
19
-
20
- from lib.renderer.mesh import load_scan, compute_tangent
21
- from lib.renderer.camera import Camera
22
- import cv2
23
- import math
24
- import random
25
- import numpy as np
26
-
27
-
28
- def render_result(rndr, shader_id, path, mask=False):
29
-
30
- cam_render = rndr.get_color(shader_id)
31
- cam_render = cv2.cvtColor(cam_render, cv2.COLOR_RGBA2BGRA)
32
-
33
- os.makedirs(os.path.dirname(path), exist_ok=True)
34
- if shader_id != 2:
35
- cv2.imwrite(path, np.uint8(255.0 * cam_render))
36
- else:
37
- cam_render[:, :, -1] -= 0.5
38
- cam_render[:, :, -1] *= 2.0
39
- if not mask:
40
- cv2.imwrite(path, np.uint8(255.0 / 2.0 * (cam_render + 1.0)))
41
- else:
42
- cv2.imwrite(path, np.uint8(-1.0 * cam_render[:, :, [3]]))
43
-
44
-
45
- def make_rotate(rx, ry, rz):
46
- sinX = np.sin(rx)
47
- sinY = np.sin(ry)
48
- sinZ = np.sin(rz)
49
-
50
- cosX = np.cos(rx)
51
- cosY = np.cos(ry)
52
- cosZ = np.cos(rz)
53
-
54
- Rx = np.zeros((3, 3))
55
- Rx[0, 0] = 1.0
56
- Rx[1, 1] = cosX
57
- Rx[1, 2] = -sinX
58
- Rx[2, 1] = sinX
59
- Rx[2, 2] = cosX
60
-
61
- Ry = np.zeros((3, 3))
62
- Ry[0, 0] = cosY
63
- Ry[0, 2] = sinY
64
- Ry[1, 1] = 1.0
65
- Ry[2, 0] = -sinY
66
- Ry[2, 2] = cosY
67
-
68
- Rz = np.zeros((3, 3))
69
- Rz[0, 0] = cosZ
70
- Rz[0, 1] = -sinZ
71
- Rz[1, 0] = sinZ
72
- Rz[1, 1] = cosZ
73
- Rz[2, 2] = 1.0
74
-
75
- R = np.matmul(np.matmul(Rz, Ry), Rx)
76
- return R
77
-
78
-
79
- def rotateSH(SH, R):
80
- SHn = SH
81
-
82
- # 1st order
83
- SHn[1] = R[1, 1] * SH[1] - R[1, 2] * SH[2] + R[1, 0] * SH[3]
84
- SHn[2] = -R[2, 1] * SH[1] + R[2, 2] * SH[2] - R[2, 0] * SH[3]
85
- SHn[3] = R[0, 1] * SH[1] - R[0, 2] * SH[2] + R[0, 0] * SH[3]
86
-
87
- # 2nd order
88
- SHn[4:, 0] = rotateBand2(SH[4:, 0], R)
89
- SHn[4:, 1] = rotateBand2(SH[4:, 1], R)
90
- SHn[4:, 2] = rotateBand2(SH[4:, 2], R)
91
-
92
- return SHn
93
-
94
-
95
- def rotateBand2(x, R):
96
- s_c3 = 0.94617469575
97
- s_c4 = -0.31539156525
98
- s_c5 = 0.54627421529
99
-
100
- s_c_scale = 1.0 / 0.91529123286551084
101
- s_c_scale_inv = 0.91529123286551084
102
-
103
- s_rc2 = 1.5853309190550713 * s_c_scale
104
- s_c4_div_c3 = s_c4 / s_c3
105
- s_c4_div_c3_x2 = (s_c4 / s_c3) * 2.0
106
-
107
- s_scale_dst2 = s_c3 * s_c_scale_inv
108
- s_scale_dst4 = s_c5 * s_c_scale_inv
109
-
110
- sh0 = x[3] + x[4] + x[4] - x[1]
111
- sh1 = x[0] + s_rc2 * x[2] + x[3] + x[4]
112
- sh2 = x[0]
113
- sh3 = -x[3]
114
- sh4 = -x[1]
115
-
116
- r2x = R[0][0] + R[0][1]
117
- r2y = R[1][0] + R[1][1]
118
- r2z = R[2][0] + R[2][1]
119
-
120
- r3x = R[0][0] + R[0][2]
121
- r3y = R[1][0] + R[1][2]
122
- r3z = R[2][0] + R[2][2]
123
-
124
- r4x = R[0][1] + R[0][2]
125
- r4y = R[1][1] + R[1][2]
126
- r4z = R[2][1] + R[2][2]
127
-
128
- sh0_x = sh0 * R[0][0]
129
- sh0_y = sh0 * R[1][0]
130
- d0 = sh0_x * R[1][0]
131
- d1 = sh0_y * R[2][0]
132
- d2 = sh0 * (R[2][0] * R[2][0] + s_c4_div_c3)
133
- d3 = sh0_x * R[2][0]
134
- d4 = sh0_x * R[0][0] - sh0_y * R[1][0]
135
-
136
- sh1_x = sh1 * R[0][2]
137
- sh1_y = sh1 * R[1][2]
138
- d0 += sh1_x * R[1][2]
139
- d1 += sh1_y * R[2][2]
140
- d2 += sh1 * (R[2][2] * R[2][2] + s_c4_div_c3)
141
- d3 += sh1_x * R[2][2]
142
- d4 += sh1_x * R[0][2] - sh1_y * R[1][2]
143
-
144
- sh2_x = sh2 * r2x
145
- sh2_y = sh2 * r2y
146
- d0 += sh2_x * r2y
147
- d1 += sh2_y * r2z
148
- d2 += sh2 * (r2z * r2z + s_c4_div_c3_x2)
149
- d3 += sh2_x * r2z
150
- d4 += sh2_x * r2x - sh2_y * r2y
151
-
152
- sh3_x = sh3 * r3x
153
- sh3_y = sh3 * r3y
154
- d0 += sh3_x * r3y
155
- d1 += sh3_y * r3z
156
- d2 += sh3 * (r3z * r3z + s_c4_div_c3_x2)
157
- d3 += sh3_x * r3z
158
- d4 += sh3_x * r3x - sh3_y * r3y
159
-
160
- sh4_x = sh4 * r4x
161
- sh4_y = sh4 * r4y
162
- d0 += sh4_x * r4y
163
- d1 += sh4_y * r4z
164
- d2 += sh4 * (r4z * r4z + s_c4_div_c3_x2)
165
- d3 += sh4_x * r4z
166
- d4 += sh4_x * r4x - sh4_y * r4y
167
-
168
- dst = x
169
- dst[0] = d0
170
- dst[1] = -d1
171
- dst[2] = d2 * s_scale_dst2
172
- dst[3] = -d3
173
- dst[4] = d4 * s_scale_dst4
174
-
175
- return dst
176
-
177
-
178
- def load_calib(param, render_size=512):
179
- # pixel unit / world unit
180
- ortho_ratio = param['ortho_ratio']
181
- # world unit / model unit
182
- scale = param['scale']
183
- # camera center world coordinate
184
- center = param['center']
185
- # model rotation
186
- R = param['R']
187
-
188
- translate = -np.matmul(R, center).reshape(3, 1)
189
- extrinsic = np.concatenate([R, translate], axis=1)
190
- extrinsic = np.concatenate(
191
- [extrinsic, np.array([0, 0, 0, 1]).reshape(1, 4)], 0)
192
- # Match camera space to image pixel space
193
- scale_intrinsic = np.identity(4)
194
- scale_intrinsic[0, 0] = scale / ortho_ratio
195
- scale_intrinsic[1, 1] = -scale / ortho_ratio
196
- scale_intrinsic[2, 2] = scale / ortho_ratio
197
- # Match image pixel space to image uv space
198
- uv_intrinsic = np.identity(4)
199
- uv_intrinsic[0, 0] = 1.0 / float(render_size // 2)
200
- uv_intrinsic[1, 1] = 1.0 / float(render_size // 2)
201
- uv_intrinsic[2, 2] = 1.0 / float(render_size // 2)
202
-
203
- intrinsic = np.matmul(uv_intrinsic, scale_intrinsic)
204
- calib = np.concatenate([extrinsic, intrinsic], axis=0)
205
- return calib
206
-
207
-
208
- def render_prt_ortho(out_path,
209
- folder_name,
210
- subject_name,
211
- shs,
212
- rndr,
213
- rndr_uv,
214
- im_size,
215
- angl_step=4,
216
- n_light=1,
217
- pitch=[0]):
218
- cam = Camera(width=im_size, height=im_size)
219
- cam.ortho_ratio = 0.4 * (512 / im_size)
220
- cam.near = -100
221
- cam.far = 100
222
- cam.sanity_check()
223
-
224
- # set path for obj, prt
225
- mesh_file = os.path.join(folder_name, subject_name + '_100k.obj')
226
- if not os.path.exists(mesh_file):
227
- print('ERROR: obj file does not exist!!', mesh_file)
228
- return
229
- prt_file = os.path.join(folder_name, 'bounce', 'bounce0.txt')
230
- if not os.path.exists(prt_file):
231
- print('ERROR: prt file does not exist!!!', prt_file)
232
- return
233
- face_prt_file = os.path.join(folder_name, 'bounce', 'face.npy')
234
- if not os.path.exists(face_prt_file):
235
- print('ERROR: face prt file does not exist!!!', prt_file)
236
- return
237
- text_file = os.path.join(folder_name, 'tex', subject_name + '_dif_2k.jpg')
238
- if not os.path.exists(text_file):
239
- print('ERROR: dif file does not exist!!', text_file)
240
- return
241
-
242
- texture_image = cv2.imread(text_file)
243
- texture_image = cv2.cvtColor(texture_image, cv2.COLOR_BGR2RGB)
244
-
245
- vertices, faces, normals, faces_normals, textures, face_textures = load_scan(
246
- mesh_file, with_normal=True, with_texture=True)
247
- vmin = vertices.min(0)
248
- vmax = vertices.max(0)
249
- up_axis = 1 if (vmax - vmin).argmax() == 1 else 2
250
-
251
- vmed = np.median(vertices, 0)
252
- vmed[up_axis] = 0.5 * (vmax[up_axis] + vmin[up_axis])
253
- y_scale = 180 / (vmax[up_axis] - vmin[up_axis])
254
-
255
- rndr.set_norm_mat(y_scale, vmed)
256
- rndr_uv.set_norm_mat(y_scale, vmed)
257
-
258
- tan, bitan = compute_tangent(vertices, faces, normals, textures,
259
- face_textures)
260
- prt = np.loadtxt(prt_file)
261
- face_prt = np.load(face_prt_file)
262
- rndr.set_mesh(vertices, faces, normals, faces_normals, textures,
263
- face_textures, prt, face_prt, tan, bitan)
264
- rndr.set_albedo(texture_image)
265
-
266
- rndr_uv.set_mesh(vertices, faces, normals, faces_normals, textures,
267
- face_textures, prt, face_prt, tan, bitan)
268
- rndr_uv.set_albedo(texture_image)
269
-
270
- os.makedirs(os.path.join(out_path, 'GEO', 'OBJ', subject_name),
271
- exist_ok=True)
272
- os.makedirs(os.path.join(out_path, 'PARAM', subject_name), exist_ok=True)
273
- os.makedirs(os.path.join(out_path, 'RENDER', subject_name), exist_ok=True)
274
- os.makedirs(os.path.join(out_path, 'MASK', subject_name), exist_ok=True)
275
- os.makedirs(os.path.join(out_path, 'UV_RENDER', subject_name),
276
- exist_ok=True)
277
- os.makedirs(os.path.join(out_path, 'UV_MASK', subject_name), exist_ok=True)
278
- os.makedirs(os.path.join(out_path, 'UV_POS', subject_name), exist_ok=True)
279
- os.makedirs(os.path.join(out_path, 'UV_NORMAL', subject_name),
280
- exist_ok=True)
281
-
282
- if not os.path.exists(os.path.join(out_path, 'val.txt')):
283
- f = open(os.path.join(out_path, 'val.txt'), 'w')
284
- f.close()
285
-
286
- # copy obj file
287
- cmd = 'cp %s %s' % (mesh_file,
288
- os.path.join(out_path, 'GEO', 'OBJ', subject_name))
289
- print(cmd)
290
- os.system(cmd)
291
-
292
- for p in pitch:
293
- for y in tqdm(range(0, 360, angl_step)):
294
- R = np.matmul(make_rotate(math.radians(p), 0, 0),
295
- make_rotate(0, math.radians(y), 0))
296
- if up_axis == 2:
297
- R = np.matmul(R, make_rotate(math.radians(90), 0, 0))
298
-
299
- rndr.rot_matrix = R
300
- rndr_uv.rot_matrix = R
301
- rndr.set_camera(cam)
302
- rndr_uv.set_camera(cam)
303
-
304
- for j in range(n_light):
305
- sh_id = random.randint(0, shs.shape[0] - 1)
306
- sh = shs[sh_id]
307
- sh_angle = 0.2 * np.pi * (random.random() - 0.5)
308
- sh = rotateSH(sh, make_rotate(0, sh_angle, 0).T)
309
-
310
- dic = {
311
- 'sh': sh,
312
- 'ortho_ratio': cam.ortho_ratio,
313
- 'scale': y_scale,
314
- 'center': vmed,
315
- 'R': R
316
- }
317
-
318
- rndr.set_sh(sh)
319
- rndr.analytic = False
320
- rndr.use_inverse_depth = False
321
- rndr.display()
322
-
323
- out_all_f = rndr.get_color(0)
324
- out_mask = out_all_f[:, :, 3]
325
- out_all_f = cv2.cvtColor(out_all_f, cv2.COLOR_RGBA2BGR)
326
-
327
- np.save(
328
- os.path.join(out_path, 'PARAM', subject_name,
329
- '%d_%d_%02d.npy' % (y, p, j)), dic)
330
- cv2.imwrite(
331
- os.path.join(out_path, 'RENDER', subject_name,
332
- '%d_%d_%02d.jpg' % (y, p, j)),
333
- 255.0 * out_all_f)
334
- cv2.imwrite(
335
- os.path.join(out_path, 'MASK', subject_name,
336
- '%d_%d_%02d.png' % (y, p, j)),
337
- 255.0 * out_mask)
338
-
339
- rndr_uv.set_sh(sh)
340
- rndr_uv.analytic = False
341
- rndr_uv.use_inverse_depth = False
342
- rndr_uv.display()
343
-
344
- uv_color = rndr_uv.get_color(0)
345
- uv_color = cv2.cvtColor(uv_color, cv2.COLOR_RGBA2BGR)
346
- cv2.imwrite(
347
- os.path.join(out_path, 'UV_RENDER', subject_name,
348
- '%d_%d_%02d.jpg' % (y, p, j)),
349
- 255.0 * uv_color)
350
-
351
- if y == 0 and j == 0 and p == pitch[0]:
352
- uv_pos = rndr_uv.get_color(1)
353
- uv_mask = uv_pos[:, :, 3]
354
- cv2.imwrite(
355
- os.path.join(out_path, 'UV_MASK', subject_name,
356
- '00.png'), 255.0 * uv_mask)
357
-
358
- data = {
359
- 'default': uv_pos[:, :, :3]
360
- } # default is a reserved name
361
- pyexr.write(
362
- os.path.join(out_path, 'UV_POS', subject_name,
363
- '00.exr'), data)
364
-
365
- uv_nml = rndr_uv.get_color(2)
366
- uv_nml = cv2.cvtColor(uv_nml, cv2.COLOR_RGBA2BGR)
367
- cv2.imwrite(
368
- os.path.join(out_path, 'UV_NORMAL', subject_name,
369
- '00.png'), 255.0 * uv_nml)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib / renderer /prt_util.py DELETED
@@ -1,199 +0,0 @@
1
-
2
- # -*- coding: utf-8 -*-
3
-
4
- # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
5
- # holder of all proprietary rights on this computer program.
6
- # You can only use this computer program if you have closed
7
- # a license agreement with MPG or you get the right to use the computer
8
- # program from someone who is authorized to grant you that right.
9
- # Any use of the computer program without a valid license is prohibited and
10
- # liable to prosecution.
11
- #
12
- # Copyright©2019 Max-Planck-Gesellschaft zur Förderung
13
- # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
14
- # for Intelligent Systems. All rights reserved.
15
- #
16
- # Contact: [email protected]
17
-
18
- import os
19
- import trimesh
20
- import numpy as np
21
- import math
22
- from scipy.special import sph_harm
23
- import argparse
24
- from tqdm import tqdm
25
- from trimesh.util import bounds_tree
26
-
27
-
28
- def factratio(N, D):
29
- if N >= D:
30
- prod = 1.0
31
- for i in range(D + 1, N + 1):
32
- prod *= i
33
- return prod
34
- else:
35
- prod = 1.0
36
- for i in range(N + 1, D + 1):
37
- prod *= i
38
- return 1.0 / prod
39
-
40
-
41
- def KVal(M, L):
42
- return math.sqrt(((2 * L + 1) / (4 * math.pi)) * (factratio(L - M, L + M)))
43
-
44
-
45
- def AssociatedLegendre(M, L, x):
46
- if M < 0 or M > L or np.max(np.abs(x)) > 1.0:
47
- return np.zeros_like(x)
48
-
49
- pmm = np.ones_like(x)
50
- if M > 0:
51
- somx2 = np.sqrt((1.0 + x) * (1.0 - x))
52
- fact = 1.0
53
- for i in range(1, M + 1):
54
- pmm = -pmm * fact * somx2
55
- fact = fact + 2
56
-
57
- if L == M:
58
- return pmm
59
- else:
60
- pmmp1 = x * (2 * M + 1) * pmm
61
- if L == M + 1:
62
- return pmmp1
63
- else:
64
- pll = np.zeros_like(x)
65
- for i in range(M + 2, L + 1):
66
- pll = (x * (2 * i - 1) * pmmp1 - (i + M - 1) * pmm) / (i - M)
67
- pmm = pmmp1
68
- pmmp1 = pll
69
- return pll
70
-
71
-
72
- def SphericalHarmonic(M, L, theta, phi):
73
- if M > 0:
74
- return math.sqrt(2.0) * KVal(M, L) * np.cos(
75
- M * phi) * AssociatedLegendre(M, L, np.cos(theta))
76
- elif M < 0:
77
- return math.sqrt(2.0) * KVal(-M, L) * np.sin(
78
- -M * phi) * AssociatedLegendre(-M, L, np.cos(theta))
79
- else:
80
- return KVal(0, L) * AssociatedLegendre(0, L, np.cos(theta))
81
-
82
-
83
- def save_obj(mesh_path, verts):
84
- file = open(mesh_path, 'w')
85
- for v in verts:
86
- file.write('v %.4f %.4f %.4f\n' % (v[0], v[1], v[2]))
87
- file.close()
88
-
89
-
90
- def sampleSphericalDirections(n):
91
- xv = np.random.rand(n, n)
92
- yv = np.random.rand(n, n)
93
- theta = np.arccos(1 - 2 * xv)
94
- phi = 2.0 * math.pi * yv
95
-
96
- phi = phi.reshape(-1)
97
- theta = theta.reshape(-1)
98
-
99
- vx = -np.sin(theta) * np.cos(phi)
100
- vy = -np.sin(theta) * np.sin(phi)
101
- vz = np.cos(theta)
102
- return np.stack([vx, vy, vz], 1), phi, theta
103
-
104
-
105
- def getSHCoeffs(order, phi, theta):
106
- shs = []
107
- for n in range(0, order + 1):
108
- for m in range(-n, n + 1):
109
- s = SphericalHarmonic(m, n, theta, phi)
110
- shs.append(s)
111
-
112
- return np.stack(shs, 1)
113
-
114
-
115
- def computePRT(mesh_path, scale, n, order):
116
-
117
- prt_dir = os.path.join(os.path.dirname(mesh_path), "prt")
118
- bounce_path = os.path.join(prt_dir, "bounce.npy")
119
- face_path = os.path.join(prt_dir, "face.npy")
120
-
121
- os.makedirs(prt_dir, exist_ok=True)
122
-
123
- PRT = None
124
- F = None
125
-
126
- if os.path.exists(bounce_path) and os.path.exists(face_path):
127
-
128
- PRT = np.load(bounce_path)
129
- F = np.load(face_path)
130
-
131
- else:
132
-
133
- mesh = trimesh.load(mesh_path,
134
- skip_materials=True,
135
- process=False,
136
- maintain_order=True)
137
- mesh.vertices *= scale
138
-
139
- vectors_orig, phi, theta = sampleSphericalDirections(n)
140
- SH_orig = getSHCoeffs(order, phi, theta)
141
-
142
- w = 4.0 * math.pi / (n * n)
143
-
144
- origins = mesh.vertices
145
- normals = mesh.vertex_normals
146
- n_v = origins.shape[0]
147
-
148
- origins = np.repeat(origins[:, None], n, axis=1).reshape(-1, 3)
149
- normals = np.repeat(normals[:, None], n, axis=1).reshape(-1, 3)
150
- PRT_all = None
151
- for i in range(n):
152
- SH = np.repeat(SH_orig[None, (i * n):((i + 1) * n)], n_v,
153
- axis=0).reshape(-1, SH_orig.shape[1])
154
- vectors = np.repeat(vectors_orig[None, (i * n):((i + 1) * n)],
155
- n_v,
156
- axis=0).reshape(-1, 3)
157
-
158
- dots = (vectors * normals).sum(1)
159
- front = (dots > 0.0)
160
-
161
- delta = 1e-3 * min(mesh.bounding_box.extents)
162
-
163
- hits = mesh.ray.intersects_any(origins + delta * normals, vectors)
164
- nohits = np.logical_and(front, np.logical_not(hits))
165
-
166
- PRT = (nohits.astype(np.float) * dots)[:, None] * SH
167
-
168
- if PRT_all is not None:
169
- PRT_all += (PRT.reshape(-1, n, SH.shape[1]).sum(1))
170
- else:
171
- PRT_all = (PRT.reshape(-1, n, SH.shape[1]).sum(1))
172
-
173
- PRT = w * PRT_all
174
- F = mesh.faces
175
-
176
- np.save(bounce_path, PRT)
177
- np.save(face_path, F)
178
-
179
- # NOTE: trimesh sometimes break the original vertex order, but topology will not change.
180
- # when loading PRT in other program, use the triangle list from trimesh.
181
-
182
- return PRT, F
183
-
184
-
185
- def testPRT(obj_path, n=40):
186
-
187
- os.makedirs(os.path.join(os.path.dirname(obj_path),
188
- f'../bounce/{os.path.basename(obj_path)[:-4]}'),
189
- exist_ok=True)
190
-
191
- PRT, F = computePRT(obj_path, n, 2)
192
- np.savetxt(
193
- os.path.join(os.path.dirname(obj_path),
194
- f'../bounce/{os.path.basename(obj_path)[:-4]}',
195
- 'bounce.npy'), PRT)
196
- np.save(
197
- os.path.join(os.path.dirname(obj_path),
198
- f'../bounce/{os.path.basename(obj_path)[:-4]}',
199
- 'face.npy'), F)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib /common / __init__.py DELETED
File without changes
lib /common / seg3d_utils.py DELETED
@@ -1,390 +0,0 @@
1
-
2
- # -*- coding: utf-8 -*-
3
-
4
- # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
5
- # holder of all proprietary rights on this computer program.
6
- # You can only use this computer program if you have closed
7
- # a license agreement with MPG or you get the right to use the computer
8
- # program from someone who is authorized to grant you that right.
9
- # Any use of the computer program without a valid license is prohibited and
10
- # liable to prosecution.
11
- #
12
- # Copyright©2019 Max-Planck-Gesellschaft zur Förderung
13
- # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
14
- # for Intelligent Systems. All rights reserved.
15
- #
16
- # Contact: [email protected]
17
-
18
- import torch
19
- import torch.nn as nn
20
- import torch.nn.functional as F
21
- import matplotlib.pyplot as plt
22
-
23
-
24
- def plot_mask2D(mask,
25
- title="",
26
- point_coords=None,
27
- figsize=10,
28
- point_marker_size=5):
29
- '''
30
- Simple plotting tool to show intermediate mask predictions and points
31
- where PointRend is applied.
32
- Args:
33
- mask (Tensor): mask prediction of shape HxW
34
- title (str): title for the plot
35
- point_coords ((Tensor, Tensor)): x and y point coordinates
36
- figsize (int): size of the figure to plot
37
- point_marker_size (int): marker size for points
38
- '''
39
-
40
- H, W = mask.shape
41
- plt.figure(figsize=(figsize, figsize))
42
- if title:
43
- title += ", "
44
- plt.title("{}resolution {}x{}".format(title, H, W), fontsize=30)
45
- plt.ylabel(H, fontsize=30)
46
- plt.xlabel(W, fontsize=30)
47
- plt.xticks([], [])
48
- plt.yticks([], [])
49
- plt.imshow(mask.detach(),
50
- interpolation="nearest",
51
- cmap=plt.get_cmap('gray'))
52
- if point_coords is not None:
53
- plt.scatter(x=point_coords[0],
54
- y=point_coords[1],
55
- color="red",
56
- s=point_marker_size,
57
- clip_on=True)
58
- plt.xlim(-0.5, W - 0.5)
59
- plt.ylim(H - 0.5, -0.5)
60
- plt.show()
61
-
62
-
63
- def plot_mask3D(mask=None,
64
- title="",
65
- point_coords=None,
66
- figsize=1500,
67
- point_marker_size=8,
68
- interactive=True):
69
- '''
70
- Simple plotting tool to show intermediate mask predictions and points
71
- where PointRend is applied.
72
- Args:
73
- mask (Tensor): mask prediction of shape DxHxW
74
- title (str): title for the plot
75
- point_coords ((Tensor, Tensor, Tensor)): x and y and z point coordinates
76
- figsize (int): size of the figure to plot
77
- point_marker_size (int): marker size for points
78
- '''
79
- import trimesh
80
- import vtkplotter
81
- from skimage import measure
82
-
83
- vp = vtkplotter.Plotter(title=title, size=(figsize, figsize))
84
- vis_list = []
85
-
86
- if mask is not None:
87
- mask = mask.detach().to("cpu").numpy()
88
- mask = mask.transpose(2, 1, 0)
89
-
90
- # marching cube to find surface
91
- verts, faces, normals, values = measure.marching_cubes_lewiner(
92
- mask, 0.5, gradient_direction='ascent')
93
-
94
- # create a mesh
95
- mesh = trimesh.Trimesh(verts, faces)
96
- mesh.visual.face_colors = [200, 200, 250, 100]
97
- vis_list.append(mesh)
98
-
99
- if point_coords is not None:
100
- point_coords = torch.stack(point_coords, 1).to("cpu").numpy()
101
-
102
- # import numpy as np
103
- # select_x = np.logical_and(point_coords[:, 0] >= 16, point_coords[:, 0] <= 112)
104
- # select_y = np.logical_and(point_coords[:, 1] >= 48, point_coords[:, 1] <= 272)
105
- # select_z = np.logical_and(point_coords[:, 2] >= 16, point_coords[:, 2] <= 112)
106
- # select = np.logical_and(np.logical_and(select_x, select_y), select_z)
107
- # point_coords = point_coords[select, :]
108
-
109
- pc = vtkplotter.Points(point_coords, r=point_marker_size, c='red')
110
- vis_list.append(pc)
111
-
112
- vp.show(*vis_list,
113
- bg="white",
114
- axes=1,
115
- interactive=interactive,
116
- azimuth=30,
117
- elevation=30)
118
-
119
-
120
- def create_grid3D(min, max, steps):
121
- if type(min) is int:
122
- min = (min, min, min) # (x, y, z)
123
- if type(max) is int:
124
- max = (max, max, max) # (x, y)
125
- if type(steps) is int:
126
- steps = (steps, steps, steps) # (x, y, z)
127
- arrangeX = torch.linspace(min[0], max[0], steps[0]).long()
128
- arrangeY = torch.linspace(min[1], max[1], steps[1]).long()
129
- arrangeZ = torch.linspace(min[2], max[2], steps[2]).long()
130
- gridD, girdH, gridW = torch.meshgrid([arrangeZ, arrangeY, arrangeX])
131
- coords = torch.stack([gridW, girdH,
132
- gridD]) # [2, steps[0], steps[1], steps[2]]
133
- coords = coords.view(3, -1).t() # [N, 3]
134
- return coords
135
-
136
-
137
- def create_grid2D(min, max, steps):
138
- if type(min) is int:
139
- min = (min, min) # (x, y)
140
- if type(max) is int:
141
- max = (max, max) # (x, y)
142
- if type(steps) is int:
143
- steps = (steps, steps) # (x, y)
144
- arrangeX = torch.linspace(min[0], max[0], steps[0]).long()
145
- arrangeY = torch.linspace(min[1], max[1], steps[1]).long()
146
- girdH, gridW = torch.meshgrid([arrangeY, arrangeX])
147
- coords = torch.stack([gridW, girdH]) # [2, steps[0], steps[1]]
148
- coords = coords.view(2, -1).t() # [N, 2]
149
- return coords
150
-
151
-
152
- class SmoothConv2D(nn.Module):
153
- def __init__(self, in_channels, out_channels, kernel_size=3):
154
- super().__init__()
155
- assert kernel_size % 2 == 1, "kernel_size for smooth_conv must be odd: {3, 5, ...}"
156
- self.padding = (kernel_size - 1) // 2
157
-
158
- weight = torch.ones(
159
- (in_channels, out_channels, kernel_size, kernel_size),
160
- dtype=torch.float32) / (kernel_size**2)
161
- self.register_buffer('weight', weight)
162
-
163
- def forward(self, input):
164
- return F.conv2d(input, self.weight, padding=self.padding)
165
-
166
-
167
- class SmoothConv3D(nn.Module):
168
- def __init__(self, in_channels, out_channels, kernel_size=3):
169
- super().__init__()
170
- assert kernel_size % 2 == 1, "kernel_size for smooth_conv must be odd: {3, 5, ...}"
171
- self.padding = (kernel_size - 1) // 2
172
-
173
- weight = torch.ones(
174
- (in_channels, out_channels, kernel_size, kernel_size, kernel_size),
175
- dtype=torch.float32) / (kernel_size**3)
176
- self.register_buffer('weight', weight)
177
-
178
- def forward(self, input):
179
- return F.conv3d(input, self.weight, padding=self.padding)
180
-
181
-
182
- def build_smooth_conv3D(in_channels=1,
183
- out_channels=1,
184
- kernel_size=3,
185
- padding=1):
186
- smooth_conv = torch.nn.Conv3d(in_channels=in_channels,
187
- out_channels=out_channels,
188
- kernel_size=kernel_size,
189
- padding=padding)
190
- smooth_conv.weight.data = torch.ones(
191
- (in_channels, out_channels, kernel_size, kernel_size, kernel_size),
192
- dtype=torch.float32) / (kernel_size**3)
193
- smooth_conv.bias.data = torch.zeros(out_channels)
194
- return smooth_conv
195
-
196
-
197
- def build_smooth_conv2D(in_channels=1,
198
- out_channels=1,
199
- kernel_size=3,
200
- padding=1):
201
- smooth_conv = torch.nn.Conv2d(in_channels=in_channels,
202
- out_channels=out_channels,
203
- kernel_size=kernel_size,
204
- padding=padding)
205
- smooth_conv.weight.data = torch.ones(
206
- (in_channels, out_channels, kernel_size, kernel_size),
207
- dtype=torch.float32) / (kernel_size**2)
208
- smooth_conv.bias.data = torch.zeros(out_channels)
209
- return smooth_conv
210
-
211
-
212
- def get_uncertain_point_coords_on_grid3D(uncertainty_map, num_points,
213
- **kwargs):
214
- """
215
- Find `num_points` most uncertain points from `uncertainty_map` grid.
216
- Args:
217
- uncertainty_map (Tensor): A tensor of shape (N, 1, H, W, D) that contains uncertainty
218
- values for a set of points on a regular H x W x D grid.
219
- num_points (int): The number of points P to select.
220
- Returns:
221
- point_indices (Tensor): A tensor of shape (N, P) that contains indices from
222
- [0, H x W x D) of the most uncertain points.
223
- point_coords (Tensor): A tensor of shape (N, P, 3) that contains [0, 1] x [0, 1] normalized
224
- coordinates of the most uncertain points from the H x W x D grid.
225
- """
226
- R, _, D, H, W = uncertainty_map.shape
227
- # h_step = 1.0 / float(H)
228
- # w_step = 1.0 / float(W)
229
- # d_step = 1.0 / float(D)
230
-
231
- num_points = min(D * H * W, num_points)
232
- point_scores, point_indices = torch.topk(uncertainty_map.view(
233
- R, D * H * W),
234
- k=num_points,
235
- dim=1)
236
- point_coords = torch.zeros(R,
237
- num_points,
238
- 3,
239
- dtype=torch.float,
240
- device=uncertainty_map.device)
241
- # point_coords[:, :, 0] = h_step / 2.0 + (point_indices // (W * D)).to(torch.float) * h_step
242
- # point_coords[:, :, 1] = w_step / 2.0 + (point_indices % (W * D) // D).to(torch.float) * w_step
243
- # point_coords[:, :, 2] = d_step / 2.0 + (point_indices % D).to(torch.float) * d_step
244
- point_coords[:, :, 0] = (point_indices % W).to(torch.float) # x
245
- point_coords[:, :, 1] = (point_indices % (H * W) // W).to(torch.float) # y
246
- point_coords[:, :, 2] = (point_indices // (H * W)).to(torch.float) # z
247
- print(f"resolution {D} x {H} x {W}", point_scores.min(),
248
- point_scores.max())
249
- return point_indices, point_coords
250
-
251
-
252
- def get_uncertain_point_coords_on_grid3D_faster(uncertainty_map, num_points,
253
- clip_min):
254
- """
255
- Find `num_points` most uncertain points from `uncertainty_map` grid.
256
- Args:
257
- uncertainty_map (Tensor): A tensor of shape (N, 1, H, W, D) that contains uncertainty
258
- values for a set of points on a regular H x W x D grid.
259
- num_points (int): The number of points P to select.
260
- Returns:
261
- point_indices (Tensor): A tensor of shape (N, P) that contains indices from
262
- [0, H x W x D) of the most uncertain points.
263
- point_coords (Tensor): A tensor of shape (N, P, 3) that contains [0, 1] x [0, 1] normalized
264
- coordinates of the most uncertain points from the H x W x D grid.
265
- """
266
- R, _, D, H, W = uncertainty_map.shape
267
- # h_step = 1.0 / float(H)
268
- # w_step = 1.0 / float(W)
269
- # d_step = 1.0 / float(D)
270
-
271
- assert R == 1, "batchsize > 1 is not implemented!"
272
- uncertainty_map = uncertainty_map.view(D * H * W)
273
- indices = (uncertainty_map >= clip_min).nonzero().squeeze(1)
274
- num_points = min(num_points, indices.size(0))
275
- point_scores, point_indices = torch.topk(uncertainty_map[indices],
276
- k=num_points,
277
- dim=0)
278
- point_indices = indices[point_indices].unsqueeze(0)
279
-
280
- point_coords = torch.zeros(R,
281
- num_points,
282
- 3,
283
- dtype=torch.float,
284
- device=uncertainty_map.device)
285
- # point_coords[:, :, 0] = h_step / 2.0 + (point_indices // (W * D)).to(torch.float) * h_step
286
- # point_coords[:, :, 1] = w_step / 2.0 + (point_indices % (W * D) // D).to(torch.float) * w_step
287
- # point_coords[:, :, 2] = d_step / 2.0 + (point_indices % D).to(torch.float) * d_step
288
- point_coords[:, :, 0] = (point_indices % W).to(torch.float) # x
289
- point_coords[:, :, 1] = (point_indices % (H * W) // W).to(torch.float) # y
290
- point_coords[:, :, 2] = (point_indices // (H * W)).to(torch.float) # z
291
- # print (f"resolution {D} x {H} x {W}", point_scores.min(), point_scores.max())
292
- return point_indices, point_coords
293
-
294
-
295
- def get_uncertain_point_coords_on_grid2D(uncertainty_map, num_points,
296
- **kwargs):
297
- """
298
- Find `num_points` most uncertain points from `uncertainty_map` grid.
299
- Args:
300
- uncertainty_map (Tensor): A tensor of shape (N, 1, H, W) that contains uncertainty
301
- values for a set of points on a regular H x W grid.
302
- num_points (int): The number of points P to select.
303
- Returns:
304
- point_indices (Tensor): A tensor of shape (N, P) that contains indices from
305
- [0, H x W) of the most uncertain points.
306
- point_coords (Tensor): A tensor of shape (N, P, 2) that contains [0, 1] x [0, 1] normalized
307
- coordinates of the most uncertain points from the H x W grid.
308
- """
309
- R, _, H, W = uncertainty_map.shape
310
- # h_step = 1.0 / float(H)
311
- # w_step = 1.0 / float(W)
312
-
313
- num_points = min(H * W, num_points)
314
- point_scores, point_indices = torch.topk(uncertainty_map.view(R, H * W),
315
- k=num_points,
316
- dim=1)
317
- point_coords = torch.zeros(R,
318
- num_points,
319
- 2,
320
- dtype=torch.long,
321
- device=uncertainty_map.device)
322
- # point_coords[:, :, 0] = w_step / 2.0 + (point_indices % W).to(torch.float) * w_step
323
- # point_coords[:, :, 1] = h_step / 2.0 + (point_indices // W).to(torch.float) * h_step
324
- point_coords[:, :, 0] = (point_indices % W).to(torch.long)
325
- point_coords[:, :, 1] = (point_indices // W).to(torch.long)
326
- # print (point_scores.min(), point_scores.max())
327
- return point_indices, point_coords
328
-
329
-
330
- def get_uncertain_point_coords_on_grid2D_faster(uncertainty_map, num_points,
331
- clip_min):
332
- """
333
- Find `num_points` most uncertain points from `uncertainty_map` grid.
334
- Args:
335
- uncertainty_map (Tensor): A tensor of shape (N, 1, H, W) that contains uncertainty
336
- values for a set of points on a regular H x W grid.
337
- num_points (int): The number of points P to select.
338
- Returns:
339
- point_indices (Tensor): A tensor of shape (N, P) that contains indices from
340
- [0, H x W) of the most uncertain points.
341
- point_coords (Tensor): A tensor of shape (N, P, 2) that contains [0, 1] x [0, 1] normalized
342
- coordinates of the most uncertain points from the H x W grid.
343
- """
344
- R, _, H, W = uncertainty_map.shape
345
- # h_step = 1.0 / float(H)
346
- # w_step = 1.0 / float(W)
347
-
348
- assert R == 1, "batchsize > 1 is not implemented!"
349
- uncertainty_map = uncertainty_map.view(H * W)
350
- indices = (uncertainty_map >= clip_min).nonzero().squeeze(1)
351
- num_points = min(num_points, indices.size(0))
352
- point_scores, point_indices = torch.topk(uncertainty_map[indices],
353
- k=num_points,
354
- dim=0)
355
- point_indices = indices[point_indices].unsqueeze(0)
356
-
357
- point_coords = torch.zeros(R,
358
- num_points,
359
- 2,
360
- dtype=torch.long,
361
- device=uncertainty_map.device)
362
- # point_coords[:, :, 0] = w_step / 2.0 + (point_indices % W).to(torch.float) * w_step
363
- # point_coords[:, :, 1] = h_step / 2.0 + (point_indices // W).to(torch.float) * h_step
364
- point_coords[:, :, 0] = (point_indices % W).to(torch.long)
365
- point_coords[:, :, 1] = (point_indices // W).to(torch.long)
366
- # print (point_scores.min(), point_scores.max())
367
- return point_indices, point_coords
368
-
369
-
370
- def calculate_uncertainty(logits, classes=None, balance_value=0.5):
371
- """
372
- We estimate uncerainty as L1 distance between 0.0 and the logit prediction in 'logits' for the
373
- foreground class in `classes`.
374
- Args:
375
- logits (Tensor): A tensor of shape (R, C, ...) or (R, 1, ...) for class-specific or
376
- class-agnostic, where R is the total number of predicted masks in all images and C is
377
- the number of foreground classes. The values are logits.
378
- classes (list): A list of length R that contains either predicted of ground truth class
379
- for eash predicted mask.
380
- Returns:
381
- scores (Tensor): A tensor of shape (R, 1, ...) that contains uncertainty scores with
382
- the most uncertain locations having the highest uncertainty score.
383
- """
384
- if logits.shape[1] == 1:
385
- gt_class_logits = logits
386
- else:
387
- gt_class_logits = logits[
388
- torch.arange(logits.shape[0], device=logits.device),
389
- classes].unsqueeze(1)
390
- return -torch.abs(gt_class_logits - balance_value)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib /common /cloth_extraction.py DELETED
@@ -1,170 +0,0 @@
1
- import numpy as np
2
- import json
3
- import os
4
- import itertools
5
- import trimesh
6
- from matplotlib.path import Path
7
- from collections import Counter
8
- from sklearn.neighbors import KNeighborsClassifier
9
-
10
-
11
- def load_segmentation(path, shape):
12
- """
13
- Get a segmentation mask for a given image
14
- Arguments:
15
- path: path to the segmentation json file
16
- shape: shape of the output mask
17
- Returns:
18
- Returns a segmentation mask
19
- """
20
- with open(path) as json_file:
21
- dict = json.load(json_file)
22
- segmentations = []
23
- for key, val in dict.items():
24
- if not key.startswith('item'):
25
- continue
26
-
27
- # Each item can have multiple polygons. Combine them to one
28
- # segmentation_coord = list(itertools.chain.from_iterable(val['segmentation']))
29
- # segmentation_coord = np.round(np.array(segmentation_coord)).astype(int)
30
-
31
- coordinates = []
32
- for segmentation_coord in val['segmentation']:
33
- # The format before is [x1,y1, x2, y2, ....]
34
- x = segmentation_coord[::2]
35
- y = segmentation_coord[1::2]
36
- xy = np.vstack((x, y)).T
37
- coordinates.append(xy)
38
-
39
- segmentations.append(
40
- {'type': val['category_name'], 'type_id': val['category_id'], 'coordinates': coordinates})
41
-
42
- return segmentations
43
-
44
-
45
- def smpl_to_recon_labels(recon, smpl, k=1):
46
- """
47
- Get the bodypart labels for the recon object by using the labels from the corresponding smpl object
48
- Arguments:
49
- recon: trimesh object (fully clothed model)
50
- shape: trimesh object (smpl model)
51
- k: number of nearest neighbours to use
52
- Returns:
53
- Returns a dictionary containing the bodypart and the corresponding indices
54
- """
55
- smpl_vert_segmentation = json.load(
56
- open(os.path.join(os.path.dirname(__file__), 'smpl_vert_segmentation.json')))
57
- n = smpl.vertices.shape[0]
58
- y = np.array([None] * n)
59
- for key, val in smpl_vert_segmentation.items():
60
- y[val] = key
61
-
62
- classifier = KNeighborsClassifier(n_neighbors=1)
63
- classifier.fit(smpl.vertices, y)
64
-
65
- y_pred = classifier.predict(recon.vertices)
66
-
67
- recon_labels = {}
68
- for key in smpl_vert_segmentation.keys():
69
- recon_labels[key] = list(np.argwhere(
70
- y_pred == key).flatten().astype(int))
71
-
72
- return recon_labels
73
-
74
-
75
- def extract_cloth(recon, segmentation, K, R, t, smpl=None):
76
- """
77
- Extract a portion of a mesh using 2d segmentation coordinates
78
- Arguments:
79
- recon: fully clothed mesh
80
- seg_coord: segmentation coordinates in 2D (NDC)
81
- K: intrinsic matrix of the projection
82
- R: rotation matrix of the projection
83
- t: translation vector of the projection
84
- Returns:
85
- Returns a submesh using the segmentation coordinates
86
- """
87
- seg_coord = segmentation['coord_normalized']
88
- mesh = trimesh.Trimesh(recon.vertices, recon.faces)
89
- extrinsic = np.zeros((3, 4))
90
- extrinsic[:3, :3] = R
91
- extrinsic[:, 3] = t
92
- P = K[:3, :3] @ extrinsic
93
-
94
- P_inv = np.linalg.pinv(P)
95
-
96
- # Each segmentation can contain multiple polygons
97
- # We need to check them separately
98
- points_so_far = []
99
- faces = recon.faces
100
- for polygon in seg_coord:
101
- n = len(polygon)
102
- coords_h = np.hstack((polygon, np.ones((n, 1))))
103
- # Apply the inverse projection on homogeneus 2D coordinates to get the corresponding 3d Coordinates
104
- XYZ = P_inv @ coords_h[:, :, None]
105
- XYZ = XYZ.reshape((XYZ.shape[0], XYZ.shape[1]))
106
- XYZ = XYZ[:, :3] / XYZ[:, 3, None]
107
-
108
- p = Path(XYZ[:, :2])
109
-
110
- grid = p.contains_points(recon.vertices[:, :2])
111
- indeces = np.argwhere(grid == True)
112
- points_so_far += list(indeces.flatten())
113
-
114
- if smpl is not None:
115
- num_verts = recon.vertices.shape[0]
116
- recon_labels = smpl_to_recon_labels(recon, smpl)
117
- body_parts_to_remove = ['rightHand', 'leftToeBase', 'leftFoot', 'rightFoot', 'head',
118
- 'leftHandIndex1', 'rightHandIndex1', 'rightToeBase', 'leftHand', 'rightHand']
119
- type = segmentation['type_id']
120
-
121
- # Remove additional bodyparts that are most likely not part of the segmentation but might intersect (e.g. hand in front of torso)
122
- # https://github.com/switchablenorms/DeepFashion2
123
- # Short sleeve clothes
124
- if type == 1 or type == 3 or type == 10:
125
- body_parts_to_remove += ['leftForeArm', 'rightForeArm']
126
- # No sleeves at all or lower body clothes
127
- elif type == 5 or type == 6 or type == 12 or type == 13 or type == 8 or type == 9:
128
- body_parts_to_remove += ['leftForeArm',
129
- 'rightForeArm', 'leftArm', 'rightArm']
130
- # Shorts
131
- elif type == 7:
132
- body_parts_to_remove += ['leftLeg', 'rightLeg',
133
- 'leftForeArm', 'rightForeArm', 'leftArm', 'rightArm']
134
-
135
- verts_to_remove = list(itertools.chain.from_iterable(
136
- [recon_labels[part] for part in body_parts_to_remove]))
137
-
138
- label_mask = np.zeros(num_verts, dtype=bool)
139
- label_mask[verts_to_remove] = True
140
-
141
- seg_mask = np.zeros(num_verts, dtype=bool)
142
- seg_mask[points_so_far] = True
143
-
144
- # Remove points that belong to other bodyparts
145
- # If a vertice in pointsSoFar is included in the bodyparts to remove, then these points should be removed
146
- extra_verts_to_remove = np.array(list(seg_mask) and list(label_mask))
147
-
148
- combine_mask = np.zeros(num_verts, dtype=bool)
149
- combine_mask[points_so_far] = True
150
- combine_mask[extra_verts_to_remove] = False
151
-
152
- all_indices = np.argwhere(combine_mask == True).flatten()
153
-
154
- i_x = np.where(np.in1d(faces[:, 0], all_indices))[0]
155
- i_y = np.where(np.in1d(faces[:, 1], all_indices))[0]
156
- i_z = np.where(np.in1d(faces[:, 2], all_indices))[0]
157
-
158
- faces_to_keep = np.array(list(set(i_x).union(i_y).union(i_z)))
159
- mask = np.zeros(len(recon.faces), dtype=bool)
160
- if len(faces_to_keep) > 0:
161
- mask[faces_to_keep] = True
162
-
163
- mesh.update_faces(mask)
164
- mesh.remove_unreferenced_vertices()
165
-
166
- # mesh.rezero()
167
-
168
- return mesh
169
-
170
- return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
lib /common /config.py DELETED
@@ -1,218 +0,0 @@
1
-
2
- # -*- coding: utf-8 -*-
3
-
4
- # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
5
- # holder of all proprietary rights on this computer program.
6
- # You can only use this computer program if you have closed
7
- # a license agreement with MPG or you get the right to use the computer
8
- # program from someone who is authorized to grant you that right.
9
- # Any use of the computer program without a valid license is prohibited and
10
- # liable to prosecution.
11
- #
12
- # Copyright©2019 Max-Planck-Gesellschaft zur Förderung
13
- # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
14
- # for Intelligent Systems. All rights reserved.
15
- #
16
- # Contact: [email protected]
17
-
18
- from yacs.config import CfgNode as CN
19
- import os
20
-
21
- _C = CN(new_allowed=True)
22
-
23
- # needed by trainer
24
- _C.name = 'default'
25
- _C.gpus = [0]
26
- _C.test_gpus = [1]
27
- _C.root = "./data/"
28
- _C.ckpt_dir = './data/ckpt/'
29
- _C.resume_path = ''
30
- _C.normal_path = ''
31
- _C.corr_path = ''
32
- _C.results_path = './data/results/'
33
- _C.projection_mode = 'orthogonal'
34
- _C.num_views = 1
35
- _C.sdf = False
36
- _C.sdf_clip = 5.0
37
-
38
- _C.lr_G = 1e-3
39
- _C.lr_C = 1e-3
40
- _C.lr_N = 2e-4
41
- _C.weight_decay = 0.0
42
- _C.momentum = 0.0
43
- _C.optim = 'RMSprop'
44
- _C.schedule = [5, 10, 15]
45
- _C.gamma = 0.1
46
-
47
- _C.overfit = False
48
- _C.resume = False
49
- _C.test_mode = False
50
- _C.test_uv = False
51
- _C.draw_geo_thres = 0.60
52
- _C.num_sanity_val_steps = 2
53
- _C.fast_dev = 0
54
- _C.get_fit = False
55
- _C.agora = False
56
- _C.optim_cloth = False
57
- _C.optim_body = False
58
- _C.mcube_res = 256
59
- _C.clean_mesh = True
60
- _C.remesh = False
61
-
62
- _C.batch_size = 4
63
- _C.num_threads = 8
64
-
65
- _C.num_epoch = 10
66
- _C.freq_plot = 0.01
67
- _C.freq_show_train = 0.1
68
- _C.freq_show_val = 0.2
69
- _C.freq_eval = 0.5
70
- _C.accu_grad_batch = 4
71
-
72
- _C.test_items = ['sv', 'mv', 'mv-fusion', 'hybrid', 'dc-pred', 'gt']
73
-
74
- _C.net = CN()
75
- _C.net.gtype = 'HGPIFuNet'
76
- _C.net.ctype = 'resnet18'
77
- _C.net.classifierIMF = 'MultiSegClassifier'
78
- _C.net.netIMF = 'resnet18'
79
- _C.net.norm = 'group'
80
- _C.net.norm_mlp = 'group'
81
- _C.net.norm_color = 'group'
82
- _C.net.hg_down = 'ave_pool'
83
- _C.net.num_views = 1
84
-
85
- # kernel_size, stride, dilation, padding
86
-
87
- _C.net.conv1 = [7, 2, 1, 3]
88
- _C.net.conv3x3 = [3, 1, 1, 1]
89
-
90
- _C.net.num_stack = 4
91
- _C.net.num_hourglass = 2
92
- _C.net.hourglass_dim = 256
93
- _C.net.voxel_dim = 32
94
- _C.net.resnet_dim = 120
95
- _C.net.mlp_dim = [320, 1024, 512, 256, 128, 1]
96
- _C.net.mlp_dim_knn = [320, 1024, 512, 256, 128, 3]
97
- _C.net.mlp_dim_color = [513, 1024, 512, 256, 128, 3]
98
- _C.net.mlp_dim_multiseg = [1088, 2048, 1024, 500]
99
- _C.net.res_layers = [2, 3, 4]
100
- _C.net.filter_dim = 256
101
- _C.net.smpl_dim = 3
102
-
103
- _C.net.cly_dim = 3
104
- _C.net.soft_dim = 64
105
- _C.net.z_size = 200.0
106
- _C.net.N_freqs = 10
107
- _C.net.geo_w = 0.1
108
- _C.net.norm_w = 0.1
109
- _C.net.dc_w = 0.1
110
- _C.net.C_cat_to_G = False
111
-
112
- _C.net.skip_hourglass = True
113
- _C.net.use_tanh = True
114
- _C.net.soft_onehot = True
115
- _C.net.no_residual = True
116
- _C.net.use_attention = False
117
-
118
- _C.net.prior_type = "sdf"
119
- _C.net.smpl_feats = ['sdf', 'cmap', 'norm', 'vis']
120
- _C.net.use_filter = True
121
- _C.net.use_cc = False
122
- _C.net.use_PE = False
123
- _C.net.use_IGR = False
124
- _C.net.in_geo = ()
125
- _C.net.in_nml = ()
126
-
127
- _C.dataset = CN()
128
- _C.dataset.root = ''
129
- _C.dataset.set_splits = [0.95, 0.04]
130
- _C.dataset.types = [
131
- "3dpeople", "axyz", "renderpeople", "renderpeople_p27", "humanalloy"
132
- ]
133
- _C.dataset.scales = [1.0, 100.0, 1.0, 1.0, 100.0 / 39.37]
134
- _C.dataset.rp_type = "pifu900"
135
- _C.dataset.th_type = 'train'
136
- _C.dataset.input_size = 512
137
- _C.dataset.rotation_num = 3
138
- _C.dataset.num_precomp = 10 # Number of segmentation classifiers
139
- _C.dataset.num_multiseg = 500 # Number of categories per classifier
140
- _C.dataset.num_knn = 10 # for loss/error
141
- _C.dataset.num_knn_dis = 20 # for accuracy
142
- _C.dataset.num_verts_max = 20000
143
- _C.dataset.zray_type = False
144
- _C.dataset.online_smpl = False
145
- _C.dataset.noise_type = ['z-trans', 'pose', 'beta']
146
- _C.dataset.noise_scale = [0.0, 0.0, 0.0]
147
- _C.dataset.num_sample_geo = 10000
148
- _C.dataset.num_sample_color = 0
149
- _C.dataset.num_sample_seg = 0
150
- _C.dataset.num_sample_knn = 10000
151
-
152
- _C.dataset.sigma_geo = 5.0
153
- _C.dataset.sigma_color = 0.10
154
- _C.dataset.sigma_seg = 0.10
155
- _C.dataset.thickness_threshold = 20.0
156
- _C.dataset.ray_sample_num = 2
157
- _C.dataset.semantic_p = False
158
- _C.dataset.remove_outlier = False
159
-
160
- _C.dataset.train_bsize = 1.0
161
- _C.dataset.val_bsize = 1.0
162
- _C.dataset.test_bsize = 1.0
163
-
164
-
165
- def get_cfg_defaults():
166
- """Get a yacs CfgNode object with default values for my_project."""
167
- # Return a clone so that the defaults will not be altered
168
- # This is for the "local variable" use pattern
169
- return _C.clone()
170
-
171
-
172
- # Alternatively, provide a way to import the defaults as
173
- # a global singleton:
174
- cfg = _C # users can `from config import cfg`
175
-
176
- # cfg = get_cfg_defaults()
177
- # cfg.merge_from_file('./configs/example.yaml')
178
-
179
- # # Now override from a list (opts could come from the command line)
180
- # opts = ['dataset.root', './data/XXXX', 'learning_rate', '1e-2']
181
- # cfg.merge_from_list(opts)
182
-
183
-
184
- def update_cfg(cfg_file):
185
- # cfg = get_cfg_defaults()
186
- _C.merge_from_file(cfg_file)
187
- # return cfg.clone()
188
- return _C
189
-
190
-
191
- def parse_args(args):
192
- cfg_file = args.cfg_file
193
- if args.cfg_file is not None:
194
- cfg = update_cfg(args.cfg_file)
195
- else:
196
- cfg = get_cfg_defaults()
197
-
198
- # if args.misc is not None:
199
- # cfg.merge_from_list(args.misc)
200
-
201
- return cfg
202
-
203
-
204
- def parse_args_extend(args):
205
- if args.resume:
206
- if not os.path.exists(args.log_dir):
207
- raise ValueError(
208
- 'Experiment are set to resume mode, but log directory does not exist.'
209
- )
210
-
211
- # load log's cfg
212
- cfg_file = os.path.join(args.log_dir, 'cfg.yaml')
213
- cfg = update_cfg(cfg_file)
214
-
215
- if args.misc is not None:
216
- cfg.merge_from_list(args.misc)
217
- else:
218
- parse_args(args)