Spaces:
Build error
Build error
Delete lib
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- lib / pymaf / configs / pymaf_config.yaml +0 -47
- lib / pymaf /core / __init__.py +0 -0
- lib / pymaf /core / train_options.py +0 -135
- lib / pymaf /core /base_trainer.py +0 -107
- lib / pymaf /core /cfgs.py +0 -100
- lib / pymaf /core /constants.py +0 -153
- lib / pymaf /core /fits_dict.py +0 -133
- lib / pymaf /core /path_config.py +0 -24
- lib / pymaf /models / __init__.py +0 -3
- lib / pymaf /models / pymaf_net.py +0 -362
- lib / pymaf /models / smpl.py +0 -92
- lib / pymaf /models /hmr.py +0 -303
- lib / pymaf /models /maf_extractor.py +0 -135
- lib / pymaf /models /res_module.py +0 -385
- lib / pymaf /utils / __init__.py +0 -0
- lib / pymaf /utils / geometry.py +0 -435
- lib / pymaf /utils / imutils.py +0 -491
- lib / pymaf /utils / streamer.py +0 -142
- lib / pymaf /utils /transforms.py +0 -78
- lib / renderer / __init__.py +0 -0
- lib / renderer / camera.py +0 -226
- lib / renderer / gl / __init__.py +0 -0
- lib / renderer / gl / data / color.fs +0 -20
- lib / renderer / gl / data /color.vs +0 -29
- lib / renderer / gl / data /normal.fs +0 -12
- lib / renderer / gl / data /normal.vs +0 -15
- lib / renderer / gl / data /prt.fs +0 -157
- lib / renderer / gl / data /prt.vs +0 -156
- lib / renderer / gl / data /prt_uv.fs +0 -141
- lib / renderer / gl / data /prt_uv.vs +0 -168
- lib / renderer / gl / data /quad.fs +0 -11
- lib / renderer / gl / data /quad.vs +0 -11
- lib / renderer / gl / framework.py +0 -95
- lib / renderer / gl / glcontext.py +0 -136
- lib / renderer / gl / init_gl.py +0 -24
- lib / renderer / gl / norm_render.py +0 -75
- lib / renderer / gl / render.py +0 -380
- lib / renderer / gl /cam_render.py +0 -80
- lib / renderer / gl /color_render.py +0 -158
- lib / renderer / gl /normal_render.py +0 -93
- lib / renderer / gl /prt_render.py +0 -450
- lib / renderer / gl /render2.py +0 -384
- lib / renderer /glm.py +0 -143
- lib / renderer /mesh.py +0 -526
- lib / renderer /opengl_util.py +0 -369
- lib / renderer /prt_util.py +0 -199
- lib /common / __init__.py +0 -0
- lib /common / seg3d_utils.py +0 -390
- lib /common /cloth_extraction.py +0 -170
- lib /common /config.py +0 -218
lib / pymaf / configs / pymaf_config.yaml
DELETED
@@ -1,47 +0,0 @@
|
|
1 |
-
SOLVER:
|
2 |
-
MAX_ITER: 500000
|
3 |
-
TYPE: Adam
|
4 |
-
BASE_LR: 0.00005
|
5 |
-
GAMMA: 0.1
|
6 |
-
STEPS: [0]
|
7 |
-
EPOCHS: [0]
|
8 |
-
DEBUG: False
|
9 |
-
LOGDIR: ''
|
10 |
-
DEVICE: cuda
|
11 |
-
NUM_WORKERS: 8
|
12 |
-
SEED_VALUE: -1
|
13 |
-
LOSS:
|
14 |
-
KP_2D_W: 300.0
|
15 |
-
KP_3D_W: 300.0
|
16 |
-
SHAPE_W: 0.06
|
17 |
-
POSE_W: 60.0
|
18 |
-
VERT_W: 0.0
|
19 |
-
INDEX_WEIGHTS: 2.0
|
20 |
-
# Loss weights for surface parts. (24 Parts)
|
21 |
-
PART_WEIGHTS: 0.3
|
22 |
-
# Loss weights for UV regression.
|
23 |
-
POINT_REGRESSION_WEIGHTS: 0.5
|
24 |
-
TRAIN:
|
25 |
-
NUM_WORKERS: 8
|
26 |
-
BATCH_SIZE: 64
|
27 |
-
PIN_MEMORY: True
|
28 |
-
TEST:
|
29 |
-
BATCH_SIZE: 32
|
30 |
-
MODEL:
|
31 |
-
PyMAF:
|
32 |
-
BACKBONE: 'res50'
|
33 |
-
MLP_DIM: [256, 128, 64, 5]
|
34 |
-
N_ITER: 3
|
35 |
-
AUX_SUPV_ON: True
|
36 |
-
DP_HEATMAP_SIZE: 56
|
37 |
-
RES_MODEL:
|
38 |
-
DECONV_WITH_BIAS: False
|
39 |
-
NUM_DECONV_LAYERS: 3
|
40 |
-
NUM_DECONV_FILTERS:
|
41 |
-
- 256
|
42 |
-
- 256
|
43 |
-
- 256
|
44 |
-
NUM_DECONV_KERNELS:
|
45 |
-
- 4
|
46 |
-
- 4
|
47 |
-
- 4
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /core / __init__.py
DELETED
File without changes
|
lib / pymaf /core / train_options.py
DELETED
@@ -1,135 +0,0 @@
|
|
1 |
-
import argparse
|
2 |
-
|
3 |
-
|
4 |
-
class TrainOptions():
|
5 |
-
def __init__(self):
|
6 |
-
self.parser = argparse.ArgumentParser()
|
7 |
-
|
8 |
-
gen = self.parser.add_argument_group('General')
|
9 |
-
gen.add_argument(
|
10 |
-
'--resume',
|
11 |
-
dest='resume',
|
12 |
-
default=False,
|
13 |
-
action='store_true',
|
14 |
-
help='Resume from checkpoint (Use latest checkpoint by default')
|
15 |
-
|
16 |
-
io = self.parser.add_argument_group('io')
|
17 |
-
io.add_argument('--log_dir',
|
18 |
-
default='logs',
|
19 |
-
help='Directory to store logs')
|
20 |
-
io.add_argument(
|
21 |
-
'--pretrained_checkpoint',
|
22 |
-
default=None,
|
23 |
-
help='Load a pretrained checkpoint at the beginning training')
|
24 |
-
|
25 |
-
train = self.parser.add_argument_group('Training Options')
|
26 |
-
train.add_argument('--num_epochs',
|
27 |
-
type=int,
|
28 |
-
default=200,
|
29 |
-
help='Total number of training epochs')
|
30 |
-
train.add_argument('--regressor',
|
31 |
-
type=str,
|
32 |
-
choices=['hmr', 'pymaf_net'],
|
33 |
-
default='pymaf_net',
|
34 |
-
help='Name of the SMPL regressor.')
|
35 |
-
train.add_argument('--cfg_file',
|
36 |
-
type=str,
|
37 |
-
default='./configs/pymaf_config.yaml',
|
38 |
-
help='config file path for PyMAF.')
|
39 |
-
train.add_argument(
|
40 |
-
'--img_res',
|
41 |
-
type=int,
|
42 |
-
default=224,
|
43 |
-
help='Rescale bounding boxes to size [img_res, img_res] before feeding them in the network'
|
44 |
-
)
|
45 |
-
train.add_argument(
|
46 |
-
'--rot_factor',
|
47 |
-
type=float,
|
48 |
-
default=30,
|
49 |
-
help='Random rotation in the range [-rot_factor, rot_factor]')
|
50 |
-
train.add_argument(
|
51 |
-
'--noise_factor',
|
52 |
-
type=float,
|
53 |
-
default=0.4,
|
54 |
-
help='Randomly multiply pixel values with factor in the range [1-noise_factor, 1+noise_factor]'
|
55 |
-
)
|
56 |
-
train.add_argument(
|
57 |
-
'--scale_factor',
|
58 |
-
type=float,
|
59 |
-
default=0.25,
|
60 |
-
help='Rescale bounding boxes by a factor of [1-scale_factor,1+scale_factor]'
|
61 |
-
)
|
62 |
-
train.add_argument(
|
63 |
-
'--openpose_train_weight',
|
64 |
-
default=0.,
|
65 |
-
help='Weight for OpenPose keypoints during training')
|
66 |
-
train.add_argument('--gt_train_weight',
|
67 |
-
default=1.,
|
68 |
-
help='Weight for GT keypoints during training')
|
69 |
-
train.add_argument('--eval_dataset',
|
70 |
-
type=str,
|
71 |
-
default='h36m-p2-mosh',
|
72 |
-
help='Name of the evaluation dataset.')
|
73 |
-
train.add_argument('--single_dataset',
|
74 |
-
default=False,
|
75 |
-
action='store_true',
|
76 |
-
help='Use a single dataset')
|
77 |
-
train.add_argument('--single_dataname',
|
78 |
-
type=str,
|
79 |
-
default='h36m',
|
80 |
-
help='Name of the single dataset.')
|
81 |
-
train.add_argument('--eval_pve',
|
82 |
-
default=False,
|
83 |
-
action='store_true',
|
84 |
-
help='evaluate PVE')
|
85 |
-
train.add_argument('--overwrite',
|
86 |
-
default=False,
|
87 |
-
action='store_true',
|
88 |
-
help='overwrite the latest checkpoint')
|
89 |
-
|
90 |
-
train.add_argument('--distributed',
|
91 |
-
action='store_true',
|
92 |
-
help='Use distributed training')
|
93 |
-
train.add_argument('--dist_backend',
|
94 |
-
default='nccl',
|
95 |
-
type=str,
|
96 |
-
help='distributed backend')
|
97 |
-
train.add_argument('--dist_url',
|
98 |
-
default='tcp://127.0.0.1:10356',
|
99 |
-
type=str,
|
100 |
-
help='url used to set up distributed training')
|
101 |
-
train.add_argument('--world_size',
|
102 |
-
default=1,
|
103 |
-
type=int,
|
104 |
-
help='number of nodes for distributed training')
|
105 |
-
train.add_argument("--local_rank", default=0, type=int)
|
106 |
-
train.add_argument('--rank',
|
107 |
-
default=0,
|
108 |
-
type=int,
|
109 |
-
help='node rank for distributed training')
|
110 |
-
train.add_argument(
|
111 |
-
'--multiprocessing_distributed',
|
112 |
-
action='store_true',
|
113 |
-
help='Use multi-processing distributed training to launch '
|
114 |
-
'N processes per node, which has N GPUs. This is the '
|
115 |
-
'fastest way to use PyTorch for either single node or '
|
116 |
-
'multi node data parallel training')
|
117 |
-
|
118 |
-
misc = self.parser.add_argument_group('Misc Options')
|
119 |
-
misc.add_argument('--misc',
|
120 |
-
help="Modify config options using the command-line",
|
121 |
-
default=None,
|
122 |
-
nargs=argparse.REMAINDER)
|
123 |
-
return
|
124 |
-
|
125 |
-
def parse_args(self):
|
126 |
-
"""Parse input arguments."""
|
127 |
-
self.args = self.parser.parse_args()
|
128 |
-
self.save_dump()
|
129 |
-
return self.args
|
130 |
-
|
131 |
-
def save_dump(self):
|
132 |
-
"""Store all argument values to a json file.
|
133 |
-
The default location is logs/expname/args.json.
|
134 |
-
"""
|
135 |
-
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /core /base_trainer.py
DELETED
@@ -1,107 +0,0 @@
|
|
1 |
-
# This script is borrowed and extended from https://github.com/nkolot/SPIN/blob/master/utils/base_trainer.py
|
2 |
-
from __future__ import division
|
3 |
-
import logging
|
4 |
-
from utils import CheckpointSaver
|
5 |
-
from tensorboardX import SummaryWriter
|
6 |
-
|
7 |
-
import torch
|
8 |
-
from tqdm import tqdm
|
9 |
-
|
10 |
-
tqdm.monitor_interval = 0
|
11 |
-
|
12 |
-
|
13 |
-
logger = logging.getLogger(__name__)
|
14 |
-
|
15 |
-
|
16 |
-
class BaseTrainer(object):
|
17 |
-
"""Base class for Trainer objects.
|
18 |
-
Takes care of checkpointing/logging/resuming training.
|
19 |
-
"""
|
20 |
-
|
21 |
-
def __init__(self, options):
|
22 |
-
self.options = options
|
23 |
-
if options.multiprocessing_distributed:
|
24 |
-
self.device = torch.device('cuda', options.gpu)
|
25 |
-
else:
|
26 |
-
self.device = torch.device(
|
27 |
-
'cuda' if torch.cuda.is_available() else 'cpu')
|
28 |
-
# override this function to define your model, optimizers etc.
|
29 |
-
self.saver = CheckpointSaver(save_dir=options.checkpoint_dir,
|
30 |
-
overwrite=options.overwrite)
|
31 |
-
if options.rank == 0:
|
32 |
-
self.summary_writer = SummaryWriter(self.options.summary_dir)
|
33 |
-
self.init_fn()
|
34 |
-
|
35 |
-
self.checkpoint = None
|
36 |
-
if options.resume and self.saver.exists_checkpoint():
|
37 |
-
self.checkpoint = self.saver.load_checkpoint(
|
38 |
-
self.models_dict, self.optimizers_dict)
|
39 |
-
|
40 |
-
if self.checkpoint is None:
|
41 |
-
self.epoch_count = 0
|
42 |
-
self.step_count = 0
|
43 |
-
else:
|
44 |
-
self.epoch_count = self.checkpoint['epoch']
|
45 |
-
self.step_count = self.checkpoint['total_step_count']
|
46 |
-
|
47 |
-
if self.checkpoint is not None:
|
48 |
-
self.checkpoint_batch_idx = self.checkpoint['batch_idx']
|
49 |
-
else:
|
50 |
-
self.checkpoint_batch_idx = 0
|
51 |
-
|
52 |
-
self.best_performance = float('inf')
|
53 |
-
|
54 |
-
def load_pretrained(self, checkpoint_file=None):
|
55 |
-
"""Load a pretrained checkpoint.
|
56 |
-
This is different from resuming training using --resume.
|
57 |
-
"""
|
58 |
-
if checkpoint_file is not None:
|
59 |
-
checkpoint = torch.load(checkpoint_file)
|
60 |
-
for model in self.models_dict:
|
61 |
-
if model in checkpoint:
|
62 |
-
self.models_dict[model].load_state_dict(checkpoint[model],
|
63 |
-
strict=True)
|
64 |
-
print(f'Checkpoint {model} loaded')
|
65 |
-
|
66 |
-
def move_dict_to_device(self, dict, device, tensor2float=False):
|
67 |
-
for k, v in dict.items():
|
68 |
-
if isinstance(v, torch.Tensor):
|
69 |
-
if tensor2float:
|
70 |
-
dict[k] = v.float().to(device)
|
71 |
-
else:
|
72 |
-
dict[k] = v.to(device)
|
73 |
-
|
74 |
-
# The following methods (with the possible exception of test) have to be implemented in the derived classes
|
75 |
-
def train(self, epoch):
|
76 |
-
raise NotImplementedError('You need to provide an train method')
|
77 |
-
|
78 |
-
def init_fn(self):
|
79 |
-
raise NotImplementedError('You need to provide an _init_fn method')
|
80 |
-
|
81 |
-
def train_step(self, input_batch):
|
82 |
-
raise NotImplementedError('You need to provide a _train_step method')
|
83 |
-
|
84 |
-
def train_summaries(self, input_batch):
|
85 |
-
raise NotImplementedError(
|
86 |
-
'You need to provide a _train_summaries method')
|
87 |
-
|
88 |
-
def visualize(self, input_batch):
|
89 |
-
raise NotImplementedError('You need to provide a visualize method')
|
90 |
-
|
91 |
-
def validate(self):
|
92 |
-
pass
|
93 |
-
|
94 |
-
def test(self):
|
95 |
-
pass
|
96 |
-
|
97 |
-
def evaluate(self):
|
98 |
-
pass
|
99 |
-
|
100 |
-
def fit(self):
|
101 |
-
# Run training for num_epochs epochs
|
102 |
-
for epoch in tqdm(range(self.epoch_count, self.options.num_epochs),
|
103 |
-
total=self.options.num_epochs,
|
104 |
-
initial=self.epoch_count):
|
105 |
-
self.epoch_count = epoch
|
106 |
-
self.train(epoch)
|
107 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /core /cfgs.py
DELETED
@@ -1,100 +0,0 @@
|
|
1 |
-
# -*- coding: utf-8 -*-
|
2 |
-
|
3 |
-
# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
|
4 |
-
# holder of all proprietary rights on this computer program.
|
5 |
-
# You can only use this computer program if you have closed
|
6 |
-
# a license agreement with MPG or you get the right to use the computer
|
7 |
-
# program from someone who is authorized to grant you that right.
|
8 |
-
# Any use of the computer program without a valid license is prohibited and
|
9 |
-
# liable to prosecution.
|
10 |
-
#
|
11 |
-
# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
|
12 |
-
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
|
13 |
-
# for Intelligent Systems. All rights reserved.
|
14 |
-
#
|
15 |
-
# Contact: [email protected]
|
16 |
-
|
17 |
-
import os
|
18 |
-
import json
|
19 |
-
from yacs.config import CfgNode as CN
|
20 |
-
|
21 |
-
# Configuration variables
|
22 |
-
cfg = CN(new_allowed=True)
|
23 |
-
|
24 |
-
cfg.OUTPUT_DIR = 'results'
|
25 |
-
cfg.DEVICE = 'cuda'
|
26 |
-
cfg.DEBUG = False
|
27 |
-
cfg.LOGDIR = ''
|
28 |
-
cfg.VAL_VIS_BATCH_FREQ = 200
|
29 |
-
cfg.TRAIN_VIS_ITER_FERQ = 1000
|
30 |
-
cfg.SEED_VALUE = -1
|
31 |
-
|
32 |
-
cfg.TRAIN = CN(new_allowed=True)
|
33 |
-
|
34 |
-
cfg.LOSS = CN(new_allowed=True)
|
35 |
-
cfg.LOSS.KP_2D_W = 300.0
|
36 |
-
cfg.LOSS.KP_3D_W = 300.0
|
37 |
-
cfg.LOSS.SHAPE_W = 0.06
|
38 |
-
cfg.LOSS.POSE_W = 60.0
|
39 |
-
cfg.LOSS.VERT_W = 0.0
|
40 |
-
|
41 |
-
# Loss weights for dense correspondences
|
42 |
-
cfg.LOSS.INDEX_WEIGHTS = 2.0
|
43 |
-
# Loss weights for surface parts. (24 Parts)
|
44 |
-
cfg.LOSS.PART_WEIGHTS = 0.3
|
45 |
-
# Loss weights for UV regression.
|
46 |
-
cfg.LOSS.POINT_REGRESSION_WEIGHTS = 0.5
|
47 |
-
|
48 |
-
cfg.MODEL = CN(new_allowed=True)
|
49 |
-
|
50 |
-
cfg.MODEL.PyMAF = CN(new_allowed=True)
|
51 |
-
|
52 |
-
# switch
|
53 |
-
cfg.TRAIN.VAL_LOOP = True
|
54 |
-
|
55 |
-
cfg.TEST = CN(new_allowed=True)
|
56 |
-
|
57 |
-
|
58 |
-
def get_cfg_defaults():
|
59 |
-
"""Get a yacs CfgNode object with default values for my_project."""
|
60 |
-
# Return a clone so that the defaults will not be altered
|
61 |
-
# This is for the "local variable" use pattern
|
62 |
-
# return cfg.clone()
|
63 |
-
return cfg
|
64 |
-
|
65 |
-
|
66 |
-
def update_cfg(cfg_file):
|
67 |
-
# cfg = get_cfg_defaults()
|
68 |
-
cfg.merge_from_file(cfg_file)
|
69 |
-
# return cfg.clone()
|
70 |
-
return cfg
|
71 |
-
|
72 |
-
|
73 |
-
def parse_args(args):
|
74 |
-
cfg_file = args.cfg_file
|
75 |
-
if args.cfg_file is not None:
|
76 |
-
cfg = update_cfg(args.cfg_file)
|
77 |
-
else:
|
78 |
-
cfg = get_cfg_defaults()
|
79 |
-
|
80 |
-
# if args.misc is not None:
|
81 |
-
# cfg.merge_from_list(args.misc)
|
82 |
-
|
83 |
-
return cfg
|
84 |
-
|
85 |
-
|
86 |
-
def parse_args_extend(args):
|
87 |
-
if args.resume:
|
88 |
-
if not os.path.exists(args.log_dir):
|
89 |
-
raise ValueError(
|
90 |
-
'Experiment are set to resume mode, but log directory does not exist.'
|
91 |
-
)
|
92 |
-
|
93 |
-
# load log's cfg
|
94 |
-
cfg_file = os.path.join(args.log_dir, 'cfg.yaml')
|
95 |
-
cfg = update_cfg(cfg_file)
|
96 |
-
|
97 |
-
if args.misc is not None:
|
98 |
-
cfg.merge_from_list(args.misc)
|
99 |
-
else:
|
100 |
-
parse_args(args)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /core /constants.py
DELETED
@@ -1,153 +0,0 @@
|
|
1 |
-
# This script is borrowed and extended from https://github.com/nkolot/SPIN/blob/master/constants.py
|
2 |
-
FOCAL_LENGTH = 5000.
|
3 |
-
IMG_RES = 224
|
4 |
-
|
5 |
-
# Mean and standard deviation for normalizing input image
|
6 |
-
IMG_NORM_MEAN = [0.485, 0.456, 0.406]
|
7 |
-
IMG_NORM_STD = [0.229, 0.224, 0.225]
|
8 |
-
"""
|
9 |
-
We create a superset of joints containing the OpenPose joints together with the ones that each dataset provides.
|
10 |
-
We keep a superset of 24 joints such that we include all joints from every dataset.
|
11 |
-
If a dataset doesn't provide annotations for a specific joint, we simply ignore it.
|
12 |
-
The joints used here are the following:
|
13 |
-
"""
|
14 |
-
JOINT_NAMES = [
|
15 |
-
# 25 OpenPose joints (in the order provided by OpenPose)
|
16 |
-
'OP Nose',
|
17 |
-
'OP Neck',
|
18 |
-
'OP RShoulder',
|
19 |
-
'OP RElbow',
|
20 |
-
'OP RWrist',
|
21 |
-
'OP LShoulder',
|
22 |
-
'OP LElbow',
|
23 |
-
'OP LWrist',
|
24 |
-
'OP MidHip',
|
25 |
-
'OP RHip',
|
26 |
-
'OP RKnee',
|
27 |
-
'OP RAnkle',
|
28 |
-
'OP LHip',
|
29 |
-
'OP LKnee',
|
30 |
-
'OP LAnkle',
|
31 |
-
'OP REye',
|
32 |
-
'OP LEye',
|
33 |
-
'OP REar',
|
34 |
-
'OP LEar',
|
35 |
-
'OP LBigToe',
|
36 |
-
'OP LSmallToe',
|
37 |
-
'OP LHeel',
|
38 |
-
'OP RBigToe',
|
39 |
-
'OP RSmallToe',
|
40 |
-
'OP RHeel',
|
41 |
-
# 24 Ground Truth joints (superset of joints from different datasets)
|
42 |
-
'Right Ankle',
|
43 |
-
'Right Knee',
|
44 |
-
'Right Hip', # 2
|
45 |
-
'Left Hip',
|
46 |
-
'Left Knee', # 4
|
47 |
-
'Left Ankle',
|
48 |
-
'Right Wrist', # 6
|
49 |
-
'Right Elbow',
|
50 |
-
'Right Shoulder', # 8
|
51 |
-
'Left Shoulder',
|
52 |
-
'Left Elbow', # 10
|
53 |
-
'Left Wrist',
|
54 |
-
'Neck (LSP)', # 12
|
55 |
-
'Top of Head (LSP)',
|
56 |
-
'Pelvis (MPII)', # 14
|
57 |
-
'Thorax (MPII)',
|
58 |
-
'Spine (H36M)', # 16
|
59 |
-
'Jaw (H36M)',
|
60 |
-
'Head (H36M)', # 18
|
61 |
-
'Nose',
|
62 |
-
'Left Eye',
|
63 |
-
'Right Eye',
|
64 |
-
'Left Ear',
|
65 |
-
'Right Ear'
|
66 |
-
]
|
67 |
-
|
68 |
-
# Dict containing the joints in numerical order
|
69 |
-
JOINT_IDS = {JOINT_NAMES[i]: i for i in range(len(JOINT_NAMES))}
|
70 |
-
|
71 |
-
# Map joints to SMPL joints
|
72 |
-
JOINT_MAP = {
|
73 |
-
'OP Nose': 24,
|
74 |
-
'OP Neck': 12,
|
75 |
-
'OP RShoulder': 17,
|
76 |
-
'OP RElbow': 19,
|
77 |
-
'OP RWrist': 21,
|
78 |
-
'OP LShoulder': 16,
|
79 |
-
'OP LElbow': 18,
|
80 |
-
'OP LWrist': 20,
|
81 |
-
'OP MidHip': 0,
|
82 |
-
'OP RHip': 2,
|
83 |
-
'OP RKnee': 5,
|
84 |
-
'OP RAnkle': 8,
|
85 |
-
'OP LHip': 1,
|
86 |
-
'OP LKnee': 4,
|
87 |
-
'OP LAnkle': 7,
|
88 |
-
'OP REye': 25,
|
89 |
-
'OP LEye': 26,
|
90 |
-
'OP REar': 27,
|
91 |
-
'OP LEar': 28,
|
92 |
-
'OP LBigToe': 29,
|
93 |
-
'OP LSmallToe': 30,
|
94 |
-
'OP LHeel': 31,
|
95 |
-
'OP RBigToe': 32,
|
96 |
-
'OP RSmallToe': 33,
|
97 |
-
'OP RHeel': 34,
|
98 |
-
'Right Ankle': 8,
|
99 |
-
'Right Knee': 5,
|
100 |
-
'Right Hip': 45,
|
101 |
-
'Left Hip': 46,
|
102 |
-
'Left Knee': 4,
|
103 |
-
'Left Ankle': 7,
|
104 |
-
'Right Wrist': 21,
|
105 |
-
'Right Elbow': 19,
|
106 |
-
'Right Shoulder': 17,
|
107 |
-
'Left Shoulder': 16,
|
108 |
-
'Left Elbow': 18,
|
109 |
-
'Left Wrist': 20,
|
110 |
-
'Neck (LSP)': 47,
|
111 |
-
'Top of Head (LSP)': 48,
|
112 |
-
'Pelvis (MPII)': 49,
|
113 |
-
'Thorax (MPII)': 50,
|
114 |
-
'Spine (H36M)': 51,
|
115 |
-
'Jaw (H36M)': 52,
|
116 |
-
'Head (H36M)': 53,
|
117 |
-
'Nose': 24,
|
118 |
-
'Left Eye': 26,
|
119 |
-
'Right Eye': 25,
|
120 |
-
'Left Ear': 28,
|
121 |
-
'Right Ear': 27
|
122 |
-
}
|
123 |
-
|
124 |
-
# Joint selectors
|
125 |
-
# Indices to get the 14 LSP joints from the 17 H36M joints
|
126 |
-
H36M_TO_J17 = [6, 5, 4, 1, 2, 3, 16, 15, 14, 11, 12, 13, 8, 10, 0, 7, 9]
|
127 |
-
H36M_TO_J14 = H36M_TO_J17[:14]
|
128 |
-
# Indices to get the 14 LSP joints from the ground truth joints
|
129 |
-
J24_TO_J17 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 18, 14, 16, 17]
|
130 |
-
J24_TO_J14 = J24_TO_J17[:14]
|
131 |
-
J24_TO_J19 = J24_TO_J17[:14] + [19, 20, 21, 22, 23]
|
132 |
-
J24_TO_JCOCO = [19, 20, 21, 22, 23, 9, 8, 10, 7, 11, 6, 3, 2, 4, 1, 5, 0]
|
133 |
-
|
134 |
-
# Permutation of SMPL pose parameters when flipping the shape
|
135 |
-
SMPL_JOINTS_FLIP_PERM = [
|
136 |
-
0, 2, 1, 3, 5, 4, 6, 8, 7, 9, 11, 10, 12, 14, 13, 15, 17, 16, 19, 18, 21,
|
137 |
-
20, 23, 22
|
138 |
-
]
|
139 |
-
SMPL_POSE_FLIP_PERM = []
|
140 |
-
for i in SMPL_JOINTS_FLIP_PERM:
|
141 |
-
SMPL_POSE_FLIP_PERM.append(3 * i)
|
142 |
-
SMPL_POSE_FLIP_PERM.append(3 * i + 1)
|
143 |
-
SMPL_POSE_FLIP_PERM.append(3 * i + 2)
|
144 |
-
# Permutation indices for the 24 ground truth joints
|
145 |
-
J24_FLIP_PERM = [
|
146 |
-
5, 4, 3, 2, 1, 0, 11, 10, 9, 8, 7, 6, 12, 13, 14, 15, 16, 17, 18, 19, 21,
|
147 |
-
20, 23, 22
|
148 |
-
]
|
149 |
-
# Permutation indices for the full set of 49 joints
|
150 |
-
J49_FLIP_PERM = [0, 1, 5, 6, 7, 2, 3, 4, 8, 12, 13, 14, 9, 10, 11, 16, 15, 18, 17, 22, 23, 24, 19, 20, 21]\
|
151 |
-
+ [25+i for i in J24_FLIP_PERM]
|
152 |
-
SMPL_J49_FLIP_PERM = [0, 1, 5, 6, 7, 2, 3, 4, 8, 12, 13, 14, 9, 10, 11, 16, 15, 18, 17, 22, 23, 24, 19, 20, 21]\
|
153 |
-
+ [25+i for i in SMPL_JOINTS_FLIP_PERM]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /core /fits_dict.py
DELETED
@@ -1,133 +0,0 @@
|
|
1 |
-
'''
|
2 |
-
This script is borrowed and extended from https://github.com/nkolot/SPIN/blob/master/train/fits_dict.py
|
3 |
-
'''
|
4 |
-
import os
|
5 |
-
import cv2
|
6 |
-
import torch
|
7 |
-
import numpy as np
|
8 |
-
from torchgeometry import angle_axis_to_rotation_matrix, rotation_matrix_to_angle_axis
|
9 |
-
|
10 |
-
from core import path_config, constants
|
11 |
-
|
12 |
-
import logging
|
13 |
-
|
14 |
-
logger = logging.getLogger(__name__)
|
15 |
-
|
16 |
-
|
17 |
-
class FitsDict():
|
18 |
-
""" Dictionary keeping track of the best fit per image in the training set """
|
19 |
-
|
20 |
-
def __init__(self, options, train_dataset):
|
21 |
-
self.options = options
|
22 |
-
self.train_dataset = train_dataset
|
23 |
-
self.fits_dict = {}
|
24 |
-
self.valid_fit_state = {}
|
25 |
-
# array used to flip SMPL pose parameters
|
26 |
-
self.flipped_parts = torch.tensor(constants.SMPL_POSE_FLIP_PERM,
|
27 |
-
dtype=torch.int64)
|
28 |
-
# Load dictionary state
|
29 |
-
for ds_name, ds in train_dataset.dataset_dict.items():
|
30 |
-
if ds_name in ['h36m']:
|
31 |
-
dict_file = os.path.join(path_config.FINAL_FITS_DIR,
|
32 |
-
ds_name + '.npy')
|
33 |
-
self.fits_dict[ds_name] = torch.from_numpy(np.load(dict_file))
|
34 |
-
self.valid_fit_state[ds_name] = torch.ones(len(
|
35 |
-
self.fits_dict[ds_name]),
|
36 |
-
dtype=torch.uint8)
|
37 |
-
else:
|
38 |
-
dict_file = os.path.join(path_config.FINAL_FITS_DIR,
|
39 |
-
ds_name + '.npz')
|
40 |
-
fits_dict = np.load(dict_file)
|
41 |
-
opt_pose = torch.from_numpy(fits_dict['pose'])
|
42 |
-
opt_betas = torch.from_numpy(fits_dict['betas'])
|
43 |
-
opt_valid_fit = torch.from_numpy(fits_dict['valid_fit']).to(
|
44 |
-
torch.uint8)
|
45 |
-
self.fits_dict[ds_name] = torch.cat([opt_pose, opt_betas],
|
46 |
-
dim=1)
|
47 |
-
self.valid_fit_state[ds_name] = opt_valid_fit
|
48 |
-
|
49 |
-
if not options.single_dataset:
|
50 |
-
for ds in train_dataset.datasets:
|
51 |
-
if ds.dataset not in ['h36m']:
|
52 |
-
ds.pose = self.fits_dict[ds.dataset][:, :72].numpy()
|
53 |
-
ds.betas = self.fits_dict[ds.dataset][:, 72:].numpy()
|
54 |
-
ds.has_smpl = self.valid_fit_state[ds.dataset].numpy()
|
55 |
-
|
56 |
-
def save(self):
|
57 |
-
""" Save dictionary state to disk """
|
58 |
-
for ds_name in self.train_dataset.dataset_dict.keys():
|
59 |
-
dict_file = os.path.join(self.options.checkpoint_dir,
|
60 |
-
ds_name + '_fits.npy')
|
61 |
-
np.save(dict_file, self.fits_dict[ds_name].cpu().numpy())
|
62 |
-
|
63 |
-
def __getitem__(self, x):
|
64 |
-
""" Retrieve dictionary entries """
|
65 |
-
dataset_name, ind, rot, is_flipped = x
|
66 |
-
batch_size = len(dataset_name)
|
67 |
-
pose = torch.zeros((batch_size, 72))
|
68 |
-
betas = torch.zeros((batch_size, 10))
|
69 |
-
for ds, i, n in zip(dataset_name, ind, range(batch_size)):
|
70 |
-
params = self.fits_dict[ds][i]
|
71 |
-
pose[n, :] = params[:72]
|
72 |
-
betas[n, :] = params[72:]
|
73 |
-
pose = pose.clone()
|
74 |
-
# Apply flipping and rotation
|
75 |
-
pose = self.flip_pose(self.rotate_pose(pose, rot), is_flipped)
|
76 |
-
betas = betas.clone()
|
77 |
-
return pose, betas
|
78 |
-
|
79 |
-
def get_vaild_state(self, dataset_name, ind):
|
80 |
-
batch_size = len(dataset_name)
|
81 |
-
valid_fit = torch.zeros(batch_size, dtype=torch.uint8)
|
82 |
-
for ds, i, n in zip(dataset_name, ind, range(batch_size)):
|
83 |
-
valid_fit[n] = self.valid_fit_state[ds][i]
|
84 |
-
valid_fit = valid_fit.clone()
|
85 |
-
return valid_fit
|
86 |
-
|
87 |
-
def __setitem__(self, x, val):
|
88 |
-
""" Update dictionary entries """
|
89 |
-
dataset_name, ind, rot, is_flipped, update = x
|
90 |
-
pose, betas = val
|
91 |
-
batch_size = len(dataset_name)
|
92 |
-
# Undo flipping and rotation
|
93 |
-
pose = self.rotate_pose(self.flip_pose(pose, is_flipped), -rot)
|
94 |
-
params = torch.cat((pose, betas), dim=-1).cpu()
|
95 |
-
for ds, i, n in zip(dataset_name, ind, range(batch_size)):
|
96 |
-
if update[n]:
|
97 |
-
self.fits_dict[ds][i] = params[n]
|
98 |
-
|
99 |
-
def flip_pose(self, pose, is_flipped):
|
100 |
-
"""flip SMPL pose parameters"""
|
101 |
-
is_flipped = is_flipped.byte()
|
102 |
-
pose_f = pose.clone()
|
103 |
-
pose_f[is_flipped, :] = pose[is_flipped][:, self.flipped_parts]
|
104 |
-
# we also negate the second and the third dimension of the axis-angle representation
|
105 |
-
pose_f[is_flipped, 1::3] *= -1
|
106 |
-
pose_f[is_flipped, 2::3] *= -1
|
107 |
-
return pose_f
|
108 |
-
|
109 |
-
def rotate_pose(self, pose, rot):
|
110 |
-
"""Rotate SMPL pose parameters by rot degrees"""
|
111 |
-
pose = pose.clone()
|
112 |
-
cos = torch.cos(-np.pi * rot / 180.)
|
113 |
-
sin = torch.sin(-np.pi * rot / 180.)
|
114 |
-
zeros = torch.zeros_like(cos)
|
115 |
-
r3 = torch.zeros(cos.shape[0], 1, 3, device=cos.device)
|
116 |
-
r3[:, 0, -1] = 1
|
117 |
-
R = torch.cat([
|
118 |
-
torch.stack([cos, -sin, zeros], dim=-1).unsqueeze(1),
|
119 |
-
torch.stack([sin, cos, zeros], dim=-1).unsqueeze(1), r3
|
120 |
-
],
|
121 |
-
dim=1)
|
122 |
-
global_pose = pose[:, :3]
|
123 |
-
global_pose_rotmat = angle_axis_to_rotation_matrix(global_pose)
|
124 |
-
global_pose_rotmat_3b3 = global_pose_rotmat[:, :3, :3]
|
125 |
-
global_pose_rotmat_3b3 = torch.matmul(R, global_pose_rotmat_3b3)
|
126 |
-
global_pose_rotmat[:, :3, :3] = global_pose_rotmat_3b3
|
127 |
-
global_pose_rotmat = global_pose_rotmat[:, :-1, :-1].cpu().numpy()
|
128 |
-
global_pose_np = np.zeros((global_pose.shape[0], 3))
|
129 |
-
for i in range(global_pose.shape[0]):
|
130 |
-
aa, _ = cv2.Rodrigues(global_pose_rotmat[i])
|
131 |
-
global_pose_np[i, :] = aa.squeeze()
|
132 |
-
pose[:, :3] = torch.from_numpy(global_pose_np).to(pose.device)
|
133 |
-
return pose
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /core /path_config.py
DELETED
@@ -1,24 +0,0 @@
|
|
1 |
-
"""
|
2 |
-
This script is borrowed and extended from https://github.com/nkolot/SPIN/blob/master/path_config.py
|
3 |
-
path configuration
|
4 |
-
This file contains definitions of useful data stuctures and the paths
|
5 |
-
for the datasets and data files necessary to run the code.
|
6 |
-
Things you need to change: *_ROOT that indicate the path to each dataset
|
7 |
-
"""
|
8 |
-
import os
|
9 |
-
from huggingface_hub import hf_hub_url, cached_download
|
10 |
-
|
11 |
-
# pymaf
|
12 |
-
pymaf_data_dir = hf_hub_url('Yuliang/PyMAF', '')
|
13 |
-
smpl_data_dir = hf_hub_url('Yuliang/SMPL', '')
|
14 |
-
SMPL_MODEL_DIR = os.path.join(smpl_data_dir, 'models/smpl')
|
15 |
-
|
16 |
-
SMPL_MEAN_PARAMS = cached_download(os.path.join(pymaf_data_dir, 'smpl_mean_params.npz'), use_auth_token=os.environ['ICON'])
|
17 |
-
MESH_DOWNSAMPLEING = cached_download(os.path.join(pymaf_data_dir, 'mesh_downsampling.npz'), use_auth_token=os.environ['ICON'])
|
18 |
-
CUBE_PARTS_FILE = cached_download(os.path.join(pymaf_data_dir, 'cube_parts.npy'), use_auth_token=os.environ['ICON'])
|
19 |
-
JOINT_REGRESSOR_TRAIN_EXTRA = cached_download(os.path.join(pymaf_data_dir, 'J_regressor_extra.npy'), use_auth_token=os.environ['ICON'])
|
20 |
-
JOINT_REGRESSOR_H36M = cached_download(os.path.join(pymaf_data_dir, 'J_regressor_h36m.npy'), use_auth_token=os.environ['ICON'])
|
21 |
-
VERTEX_TEXTURE_FILE = cached_download(os.path.join(pymaf_data_dir, 'vertex_texture.npy'), use_auth_token=os.environ['ICON'])
|
22 |
-
SMPL_MEAN_PARAMS = cached_download(os.path.join(pymaf_data_dir, 'smpl_mean_params.npz'), use_auth_token=os.environ['ICON'])
|
23 |
-
CHECKPOINT_FILE = cached_download(os.path.join(pymaf_data_dir, 'pretrained_model/PyMAF_model_checkpoint.pt'), use_auth_token=os.environ['ICON'])
|
24 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /models / __init__.py
DELETED
@@ -1,3 +0,0 @@
|
|
1 |
-
from .hmr import hmr
|
2 |
-
from .pymaf_net import pymaf_net
|
3 |
-
from .smpl import SMPL
|
|
|
|
|
|
|
|
lib / pymaf /models / pymaf_net.py
DELETED
@@ -1,362 +0,0 @@
|
|
1 |
-
import torch
|
2 |
-
import torch.nn as nn
|
3 |
-
import numpy as np
|
4 |
-
|
5 |
-
from lib.pymaf.utils.geometry import rot6d_to_rotmat, projection, rotation_matrix_to_angle_axis
|
6 |
-
from .maf_extractor import MAF_Extractor
|
7 |
-
from .smpl import SMPL, SMPL_MODEL_DIR, SMPL_MEAN_PARAMS, H36M_TO_J14
|
8 |
-
from .hmr import ResNet_Backbone
|
9 |
-
from .res_module import IUV_predict_layer
|
10 |
-
from lib.common.config import cfg
|
11 |
-
import logging
|
12 |
-
|
13 |
-
logger = logging.getLogger(__name__)
|
14 |
-
|
15 |
-
BN_MOMENTUM = 0.1
|
16 |
-
|
17 |
-
|
18 |
-
class Regressor(nn.Module):
|
19 |
-
def __init__(self, feat_dim, smpl_mean_params):
|
20 |
-
super().__init__()
|
21 |
-
|
22 |
-
npose = 24 * 6
|
23 |
-
|
24 |
-
self.fc1 = nn.Linear(feat_dim + npose + 13, 1024)
|
25 |
-
self.drop1 = nn.Dropout()
|
26 |
-
self.fc2 = nn.Linear(1024, 1024)
|
27 |
-
self.drop2 = nn.Dropout()
|
28 |
-
self.decpose = nn.Linear(1024, npose)
|
29 |
-
self.decshape = nn.Linear(1024, 10)
|
30 |
-
self.deccam = nn.Linear(1024, 3)
|
31 |
-
nn.init.xavier_uniform_(self.decpose.weight, gain=0.01)
|
32 |
-
nn.init.xavier_uniform_(self.decshape.weight, gain=0.01)
|
33 |
-
nn.init.xavier_uniform_(self.deccam.weight, gain=0.01)
|
34 |
-
|
35 |
-
self.smpl = SMPL(SMPL_MODEL_DIR, batch_size=64, create_transl=False)
|
36 |
-
|
37 |
-
mean_params = np.load(smpl_mean_params)
|
38 |
-
init_pose = torch.from_numpy(mean_params['pose'][:]).unsqueeze(0)
|
39 |
-
init_shape = torch.from_numpy(
|
40 |
-
mean_params['shape'][:].astype('float32')).unsqueeze(0)
|
41 |
-
init_cam = torch.from_numpy(mean_params['cam']).unsqueeze(0)
|
42 |
-
self.register_buffer('init_pose', init_pose)
|
43 |
-
self.register_buffer('init_shape', init_shape)
|
44 |
-
self.register_buffer('init_cam', init_cam)
|
45 |
-
|
46 |
-
def forward(self,
|
47 |
-
x,
|
48 |
-
init_pose=None,
|
49 |
-
init_shape=None,
|
50 |
-
init_cam=None,
|
51 |
-
n_iter=1,
|
52 |
-
J_regressor=None):
|
53 |
-
batch_size = x.shape[0]
|
54 |
-
|
55 |
-
if init_pose is None:
|
56 |
-
init_pose = self.init_pose.expand(batch_size, -1)
|
57 |
-
if init_shape is None:
|
58 |
-
init_shape = self.init_shape.expand(batch_size, -1)
|
59 |
-
if init_cam is None:
|
60 |
-
init_cam = self.init_cam.expand(batch_size, -1)
|
61 |
-
|
62 |
-
pred_pose = init_pose
|
63 |
-
pred_shape = init_shape
|
64 |
-
pred_cam = init_cam
|
65 |
-
for i in range(n_iter):
|
66 |
-
xc = torch.cat([x, pred_pose, pred_shape, pred_cam], 1)
|
67 |
-
xc = self.fc1(xc)
|
68 |
-
xc = self.drop1(xc)
|
69 |
-
xc = self.fc2(xc)
|
70 |
-
xc = self.drop2(xc)
|
71 |
-
pred_pose = self.decpose(xc) + pred_pose
|
72 |
-
pred_shape = self.decshape(xc) + pred_shape
|
73 |
-
pred_cam = self.deccam(xc) + pred_cam
|
74 |
-
|
75 |
-
pred_rotmat = rot6d_to_rotmat(pred_pose).view(batch_size, 24, 3, 3)
|
76 |
-
|
77 |
-
pred_output = self.smpl(betas=pred_shape,
|
78 |
-
body_pose=pred_rotmat[:, 1:],
|
79 |
-
global_orient=pred_rotmat[:, 0].unsqueeze(1),
|
80 |
-
pose2rot=False)
|
81 |
-
|
82 |
-
pred_vertices = pred_output.vertices
|
83 |
-
pred_joints = pred_output.joints
|
84 |
-
pred_smpl_joints = pred_output.smpl_joints
|
85 |
-
pred_keypoints_2d = projection(pred_joints, pred_cam)
|
86 |
-
pose = rotation_matrix_to_angle_axis(pred_rotmat.reshape(-1, 3,
|
87 |
-
3)).reshape(
|
88 |
-
-1, 72)
|
89 |
-
|
90 |
-
if J_regressor is not None:
|
91 |
-
pred_joints = torch.matmul(J_regressor, pred_vertices)
|
92 |
-
pred_pelvis = pred_joints[:, [0], :].clone()
|
93 |
-
pred_joints = pred_joints[:, H36M_TO_J14, :]
|
94 |
-
pred_joints = pred_joints - pred_pelvis
|
95 |
-
|
96 |
-
output = {
|
97 |
-
'theta': torch.cat([pred_cam, pred_shape, pose], dim=1),
|
98 |
-
'verts': pred_vertices,
|
99 |
-
'kp_2d': pred_keypoints_2d,
|
100 |
-
'kp_3d': pred_joints,
|
101 |
-
'smpl_kp_3d': pred_smpl_joints,
|
102 |
-
'rotmat': pred_rotmat,
|
103 |
-
'pred_cam': pred_cam,
|
104 |
-
'pred_shape': pred_shape,
|
105 |
-
'pred_pose': pred_pose,
|
106 |
-
}
|
107 |
-
return output
|
108 |
-
|
109 |
-
def forward_init(self,
|
110 |
-
x,
|
111 |
-
init_pose=None,
|
112 |
-
init_shape=None,
|
113 |
-
init_cam=None,
|
114 |
-
n_iter=1,
|
115 |
-
J_regressor=None):
|
116 |
-
batch_size = x.shape[0]
|
117 |
-
|
118 |
-
if init_pose is None:
|
119 |
-
init_pose = self.init_pose.expand(batch_size, -1)
|
120 |
-
if init_shape is None:
|
121 |
-
init_shape = self.init_shape.expand(batch_size, -1)
|
122 |
-
if init_cam is None:
|
123 |
-
init_cam = self.init_cam.expand(batch_size, -1)
|
124 |
-
|
125 |
-
pred_pose = init_pose
|
126 |
-
pred_shape = init_shape
|
127 |
-
pred_cam = init_cam
|
128 |
-
|
129 |
-
pred_rotmat = rot6d_to_rotmat(pred_pose.contiguous()).view(
|
130 |
-
batch_size, 24, 3, 3)
|
131 |
-
|
132 |
-
pred_output = self.smpl(betas=pred_shape,
|
133 |
-
body_pose=pred_rotmat[:, 1:],
|
134 |
-
global_orient=pred_rotmat[:, 0].unsqueeze(1),
|
135 |
-
pose2rot=False)
|
136 |
-
|
137 |
-
pred_vertices = pred_output.vertices
|
138 |
-
pred_joints = pred_output.joints
|
139 |
-
pred_smpl_joints = pred_output.smpl_joints
|
140 |
-
pred_keypoints_2d = projection(pred_joints, pred_cam)
|
141 |
-
pose = rotation_matrix_to_angle_axis(pred_rotmat.reshape(-1, 3,
|
142 |
-
3)).reshape(
|
143 |
-
-1, 72)
|
144 |
-
|
145 |
-
if J_regressor is not None:
|
146 |
-
pred_joints = torch.matmul(J_regressor, pred_vertices)
|
147 |
-
pred_pelvis = pred_joints[:, [0], :].clone()
|
148 |
-
pred_joints = pred_joints[:, H36M_TO_J14, :]
|
149 |
-
pred_joints = pred_joints - pred_pelvis
|
150 |
-
|
151 |
-
output = {
|
152 |
-
'theta': torch.cat([pred_cam, pred_shape, pose], dim=1),
|
153 |
-
'verts': pred_vertices,
|
154 |
-
'kp_2d': pred_keypoints_2d,
|
155 |
-
'kp_3d': pred_joints,
|
156 |
-
'smpl_kp_3d': pred_smpl_joints,
|
157 |
-
'rotmat': pred_rotmat,
|
158 |
-
'pred_cam': pred_cam,
|
159 |
-
'pred_shape': pred_shape,
|
160 |
-
'pred_pose': pred_pose,
|
161 |
-
}
|
162 |
-
return output
|
163 |
-
|
164 |
-
|
165 |
-
class PyMAF(nn.Module):
|
166 |
-
""" PyMAF based Deep Regressor for Human Mesh Recovery
|
167 |
-
PyMAF: 3D Human Pose and Shape Regression with Pyramidal Mesh Alignment Feedback Loop, in ICCV, 2021
|
168 |
-
"""
|
169 |
-
|
170 |
-
def __init__(self, smpl_mean_params=SMPL_MEAN_PARAMS, pretrained=True):
|
171 |
-
super().__init__()
|
172 |
-
self.feature_extractor = ResNet_Backbone(
|
173 |
-
model=cfg.MODEL.PyMAF.BACKBONE, pretrained=pretrained)
|
174 |
-
|
175 |
-
# deconv layers
|
176 |
-
self.inplanes = self.feature_extractor.inplanes
|
177 |
-
self.deconv_with_bias = cfg.RES_MODEL.DECONV_WITH_BIAS
|
178 |
-
self.deconv_layers = self._make_deconv_layer(
|
179 |
-
cfg.RES_MODEL.NUM_DECONV_LAYERS,
|
180 |
-
cfg.RES_MODEL.NUM_DECONV_FILTERS,
|
181 |
-
cfg.RES_MODEL.NUM_DECONV_KERNELS,
|
182 |
-
)
|
183 |
-
|
184 |
-
self.maf_extractor = nn.ModuleList()
|
185 |
-
for _ in range(cfg.MODEL.PyMAF.N_ITER):
|
186 |
-
self.maf_extractor.append(MAF_Extractor())
|
187 |
-
ma_feat_len = self.maf_extractor[-1].Dmap.shape[
|
188 |
-
0] * cfg.MODEL.PyMAF.MLP_DIM[-1]
|
189 |
-
|
190 |
-
grid_size = 21
|
191 |
-
xv, yv = torch.meshgrid([
|
192 |
-
torch.linspace(-1, 1, grid_size),
|
193 |
-
torch.linspace(-1, 1, grid_size)
|
194 |
-
])
|
195 |
-
points_grid = torch.stack([xv.reshape(-1),
|
196 |
-
yv.reshape(-1)]).unsqueeze(0)
|
197 |
-
self.register_buffer('points_grid', points_grid)
|
198 |
-
grid_feat_len = grid_size * grid_size * cfg.MODEL.PyMAF.MLP_DIM[-1]
|
199 |
-
|
200 |
-
self.regressor = nn.ModuleList()
|
201 |
-
for i in range(cfg.MODEL.PyMAF.N_ITER):
|
202 |
-
if i == 0:
|
203 |
-
ref_infeat_dim = grid_feat_len
|
204 |
-
else:
|
205 |
-
ref_infeat_dim = ma_feat_len
|
206 |
-
self.regressor.append(
|
207 |
-
Regressor(feat_dim=ref_infeat_dim,
|
208 |
-
smpl_mean_params=smpl_mean_params))
|
209 |
-
|
210 |
-
dp_feat_dim = 256
|
211 |
-
self.with_uv = cfg.LOSS.POINT_REGRESSION_WEIGHTS > 0
|
212 |
-
if cfg.MODEL.PyMAF.AUX_SUPV_ON:
|
213 |
-
self.dp_head = IUV_predict_layer(feat_dim=dp_feat_dim)
|
214 |
-
|
215 |
-
def _make_layer(self, block, planes, blocks, stride=1):
|
216 |
-
downsample = None
|
217 |
-
if stride != 1 or self.inplanes != planes * block.expansion:
|
218 |
-
downsample = nn.Sequential(
|
219 |
-
nn.Conv2d(self.inplanes,
|
220 |
-
planes * block.expansion,
|
221 |
-
kernel_size=1,
|
222 |
-
stride=stride,
|
223 |
-
bias=False),
|
224 |
-
nn.BatchNorm2d(planes * block.expansion),
|
225 |
-
)
|
226 |
-
|
227 |
-
layers = []
|
228 |
-
layers.append(block(self.inplanes, planes, stride, downsample))
|
229 |
-
self.inplanes = planes * block.expansion
|
230 |
-
for i in range(1, blocks):
|
231 |
-
layers.append(block(self.inplanes, planes))
|
232 |
-
|
233 |
-
return nn.Sequential(*layers)
|
234 |
-
|
235 |
-
def _make_deconv_layer(self, num_layers, num_filters, num_kernels):
|
236 |
-
"""
|
237 |
-
Deconv_layer used in Simple Baselines:
|
238 |
-
Xiao et al. Simple Baselines for Human Pose Estimation and Tracking
|
239 |
-
https://github.com/microsoft/human-pose-estimation.pytorch
|
240 |
-
"""
|
241 |
-
assert num_layers == len(num_filters), \
|
242 |
-
'ERROR: num_deconv_layers is different len(num_deconv_filters)'
|
243 |
-
assert num_layers == len(num_kernels), \
|
244 |
-
'ERROR: num_deconv_layers is different len(num_deconv_filters)'
|
245 |
-
|
246 |
-
def _get_deconv_cfg(deconv_kernel, index):
|
247 |
-
if deconv_kernel == 4:
|
248 |
-
padding = 1
|
249 |
-
output_padding = 0
|
250 |
-
elif deconv_kernel == 3:
|
251 |
-
padding = 1
|
252 |
-
output_padding = 1
|
253 |
-
elif deconv_kernel == 2:
|
254 |
-
padding = 0
|
255 |
-
output_padding = 0
|
256 |
-
|
257 |
-
return deconv_kernel, padding, output_padding
|
258 |
-
|
259 |
-
layers = []
|
260 |
-
for i in range(num_layers):
|
261 |
-
kernel, padding, output_padding = _get_deconv_cfg(
|
262 |
-
num_kernels[i], i)
|
263 |
-
|
264 |
-
planes = num_filters[i]
|
265 |
-
layers.append(
|
266 |
-
nn.ConvTranspose2d(in_channels=self.inplanes,
|
267 |
-
out_channels=planes,
|
268 |
-
kernel_size=kernel,
|
269 |
-
stride=2,
|
270 |
-
padding=padding,
|
271 |
-
output_padding=output_padding,
|
272 |
-
bias=self.deconv_with_bias))
|
273 |
-
layers.append(nn.BatchNorm2d(planes, momentum=BN_MOMENTUM))
|
274 |
-
layers.append(nn.ReLU(inplace=True))
|
275 |
-
self.inplanes = planes
|
276 |
-
|
277 |
-
return nn.Sequential(*layers)
|
278 |
-
|
279 |
-
def forward(self, x, J_regressor=None):
|
280 |
-
|
281 |
-
batch_size = x.shape[0]
|
282 |
-
|
283 |
-
# spatial features and global features
|
284 |
-
s_feat, g_feat = self.feature_extractor(x)
|
285 |
-
|
286 |
-
assert cfg.MODEL.PyMAF.N_ITER >= 0 and cfg.MODEL.PyMAF.N_ITER <= 3
|
287 |
-
if cfg.MODEL.PyMAF.N_ITER == 1:
|
288 |
-
deconv_blocks = [self.deconv_layers]
|
289 |
-
elif cfg.MODEL.PyMAF.N_ITER == 2:
|
290 |
-
deconv_blocks = [self.deconv_layers[0:6], self.deconv_layers[6:9]]
|
291 |
-
elif cfg.MODEL.PyMAF.N_ITER == 3:
|
292 |
-
deconv_blocks = [
|
293 |
-
self.deconv_layers[0:3], self.deconv_layers[3:6],
|
294 |
-
self.deconv_layers[6:9]
|
295 |
-
]
|
296 |
-
|
297 |
-
out_list = {}
|
298 |
-
|
299 |
-
# initial parameters
|
300 |
-
# TODO: remove the initial mesh generation during forward to reduce runtime
|
301 |
-
# by generating initial mesh the beforehand: smpl_output = self.init_smpl
|
302 |
-
smpl_output = self.regressor[0].forward_init(g_feat,
|
303 |
-
J_regressor=J_regressor)
|
304 |
-
|
305 |
-
out_list['smpl_out'] = [smpl_output]
|
306 |
-
out_list['dp_out'] = []
|
307 |
-
|
308 |
-
# for visulization
|
309 |
-
vis_feat_list = [s_feat.detach()]
|
310 |
-
|
311 |
-
# parameter predictions
|
312 |
-
for rf_i in range(cfg.MODEL.PyMAF.N_ITER):
|
313 |
-
pred_cam = smpl_output['pred_cam']
|
314 |
-
pred_shape = smpl_output['pred_shape']
|
315 |
-
pred_pose = smpl_output['pred_pose']
|
316 |
-
|
317 |
-
pred_cam = pred_cam.detach()
|
318 |
-
pred_shape = pred_shape.detach()
|
319 |
-
pred_pose = pred_pose.detach()
|
320 |
-
|
321 |
-
s_feat_i = deconv_blocks[rf_i](s_feat)
|
322 |
-
s_feat = s_feat_i
|
323 |
-
vis_feat_list.append(s_feat_i.detach())
|
324 |
-
|
325 |
-
self.maf_extractor[rf_i].im_feat = s_feat_i
|
326 |
-
self.maf_extractor[rf_i].cam = pred_cam
|
327 |
-
|
328 |
-
if rf_i == 0:
|
329 |
-
sample_points = torch.transpose(
|
330 |
-
self.points_grid.expand(batch_size, -1, -1), 1, 2)
|
331 |
-
ref_feature = self.maf_extractor[rf_i].sampling(sample_points)
|
332 |
-
else:
|
333 |
-
pred_smpl_verts = smpl_output['verts'].detach()
|
334 |
-
# TODO: use a more sparse SMPL implementation (with 431 vertices) for acceleration
|
335 |
-
pred_smpl_verts_ds = torch.matmul(
|
336 |
-
self.maf_extractor[rf_i].Dmap.unsqueeze(0),
|
337 |
-
pred_smpl_verts) # [B, 431, 3]
|
338 |
-
ref_feature = self.maf_extractor[rf_i](
|
339 |
-
pred_smpl_verts_ds) # [B, 431 * n_feat]
|
340 |
-
|
341 |
-
smpl_output = self.regressor[rf_i](ref_feature,
|
342 |
-
pred_pose,
|
343 |
-
pred_shape,
|
344 |
-
pred_cam,
|
345 |
-
n_iter=1,
|
346 |
-
J_regressor=J_regressor)
|
347 |
-
out_list['smpl_out'].append(smpl_output)
|
348 |
-
|
349 |
-
if self.training and cfg.MODEL.PyMAF.AUX_SUPV_ON:
|
350 |
-
iuv_out_dict = self.dp_head(s_feat)
|
351 |
-
out_list['dp_out'].append(iuv_out_dict)
|
352 |
-
|
353 |
-
return out_list
|
354 |
-
|
355 |
-
|
356 |
-
def pymaf_net(smpl_mean_params, pretrained=True):
|
357 |
-
""" Constructs an PyMAF model with ResNet50 backbone.
|
358 |
-
Args:
|
359 |
-
pretrained (bool): If True, returns a model pre-trained on ImageNet
|
360 |
-
"""
|
361 |
-
model = PyMAF(smpl_mean_params, pretrained)
|
362 |
-
return model
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /models / smpl.py
DELETED
@@ -1,92 +0,0 @@
|
|
1 |
-
# This script is borrowed from https://github.com/nkolot/SPIN/blob/master/models/smpl.py
|
2 |
-
|
3 |
-
import torch
|
4 |
-
import numpy as np
|
5 |
-
from lib.smplx import SMPL as _SMPL
|
6 |
-
from lib.smplx.body_models import ModelOutput
|
7 |
-
from lib.smplx.lbs import vertices2joints
|
8 |
-
from collections import namedtuple
|
9 |
-
|
10 |
-
from lib.pymaf.core import path_config, constants
|
11 |
-
|
12 |
-
SMPL_MEAN_PARAMS = path_config.SMPL_MEAN_PARAMS
|
13 |
-
SMPL_MODEL_DIR = path_config.SMPL_MODEL_DIR
|
14 |
-
|
15 |
-
# Indices to get the 14 LSP joints from the 17 H36M joints
|
16 |
-
H36M_TO_J17 = [6, 5, 4, 1, 2, 3, 16, 15, 14, 11, 12, 13, 8, 10, 0, 7, 9]
|
17 |
-
H36M_TO_J14 = H36M_TO_J17[:14]
|
18 |
-
|
19 |
-
|
20 |
-
class SMPL(_SMPL):
|
21 |
-
""" Extension of the official SMPL implementation to support more joints """
|
22 |
-
|
23 |
-
def __init__(self, *args, **kwargs):
|
24 |
-
super().__init__(*args, **kwargs)
|
25 |
-
joints = [constants.JOINT_MAP[i] for i in constants.JOINT_NAMES]
|
26 |
-
J_regressor_extra = np.load(path_config.JOINT_REGRESSOR_TRAIN_EXTRA)
|
27 |
-
self.register_buffer(
|
28 |
-
'J_regressor_extra',
|
29 |
-
torch.tensor(J_regressor_extra, dtype=torch.float32))
|
30 |
-
self.joint_map = torch.tensor(joints, dtype=torch.long)
|
31 |
-
self.ModelOutput = namedtuple(
|
32 |
-
'ModelOutput_', ModelOutput._fields + (
|
33 |
-
'smpl_joints',
|
34 |
-
'joints_J19',
|
35 |
-
))
|
36 |
-
self.ModelOutput.__new__.__defaults__ = (None, ) * len(
|
37 |
-
self.ModelOutput._fields)
|
38 |
-
|
39 |
-
def forward(self, *args, **kwargs):
|
40 |
-
kwargs['get_skin'] = True
|
41 |
-
smpl_output = super().forward(*args, **kwargs)
|
42 |
-
extra_joints = vertices2joints(self.J_regressor_extra,
|
43 |
-
smpl_output.vertices)
|
44 |
-
# smpl_output.joints: [B, 45, 3] extra_joints: [B, 9, 3]
|
45 |
-
vertices = smpl_output.vertices
|
46 |
-
joints = torch.cat([smpl_output.joints, extra_joints], dim=1)
|
47 |
-
smpl_joints = smpl_output.joints[:, :24]
|
48 |
-
joints = joints[:, self.joint_map, :] # [B, 49, 3]
|
49 |
-
joints_J24 = joints[:, -24:, :]
|
50 |
-
joints_J19 = joints_J24[:, constants.J24_TO_J19, :]
|
51 |
-
output = self.ModelOutput(vertices=vertices,
|
52 |
-
global_orient=smpl_output.global_orient,
|
53 |
-
body_pose=smpl_output.body_pose,
|
54 |
-
joints=joints,
|
55 |
-
joints_J19=joints_J19,
|
56 |
-
smpl_joints=smpl_joints,
|
57 |
-
betas=smpl_output.betas,
|
58 |
-
full_pose=smpl_output.full_pose)
|
59 |
-
return output
|
60 |
-
|
61 |
-
|
62 |
-
def get_smpl_faces():
|
63 |
-
smpl = SMPL(SMPL_MODEL_DIR, batch_size=1, create_transl=False)
|
64 |
-
return smpl.faces
|
65 |
-
|
66 |
-
|
67 |
-
def get_part_joints(smpl_joints):
|
68 |
-
batch_size = smpl_joints.shape[0]
|
69 |
-
|
70 |
-
# part_joints = torch.zeros().to(smpl_joints.device)
|
71 |
-
|
72 |
-
one_seg_pairs = [(0, 1), (0, 2), (0, 3), (3, 6), (9, 12), (9, 13), (9, 14),
|
73 |
-
(12, 15), (13, 16), (14, 17)]
|
74 |
-
two_seg_pairs = [(1, 4), (2, 5), (4, 7), (5, 8), (16, 18), (17, 19),
|
75 |
-
(18, 20), (19, 21)]
|
76 |
-
|
77 |
-
one_seg_pairs.extend(two_seg_pairs)
|
78 |
-
|
79 |
-
single_joints = [(10), (11), (15), (22), (23)]
|
80 |
-
|
81 |
-
part_joints = []
|
82 |
-
|
83 |
-
for j_p in one_seg_pairs:
|
84 |
-
new_joint = torch.mean(smpl_joints[:, j_p], dim=1, keepdim=True)
|
85 |
-
part_joints.append(new_joint)
|
86 |
-
|
87 |
-
for j_p in single_joints:
|
88 |
-
part_joints.append(smpl_joints[:, j_p:j_p + 1])
|
89 |
-
|
90 |
-
part_joints = torch.cat(part_joints, dim=1)
|
91 |
-
|
92 |
-
return part_joints
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /models /hmr.py
DELETED
@@ -1,303 +0,0 @@
|
|
1 |
-
# This script is borrowed from https://github.com/nkolot/SPIN/blob/master/models/hmr.py
|
2 |
-
|
3 |
-
import torch
|
4 |
-
import torch.nn as nn
|
5 |
-
import torchvision.models.resnet as resnet
|
6 |
-
import numpy as np
|
7 |
-
import math
|
8 |
-
from lib.pymaf.utils.geometry import rot6d_to_rotmat
|
9 |
-
|
10 |
-
import logging
|
11 |
-
|
12 |
-
logger = logging.getLogger(__name__)
|
13 |
-
|
14 |
-
BN_MOMENTUM = 0.1
|
15 |
-
|
16 |
-
|
17 |
-
class Bottleneck(nn.Module):
|
18 |
-
""" Redefinition of Bottleneck residual block
|
19 |
-
Adapted from the official PyTorch implementation
|
20 |
-
"""
|
21 |
-
expansion = 4
|
22 |
-
|
23 |
-
def __init__(self, inplanes, planes, stride=1, downsample=None):
|
24 |
-
super().__init__()
|
25 |
-
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
|
26 |
-
self.bn1 = nn.BatchNorm2d(planes)
|
27 |
-
self.conv2 = nn.Conv2d(planes,
|
28 |
-
planes,
|
29 |
-
kernel_size=3,
|
30 |
-
stride=stride,
|
31 |
-
padding=1,
|
32 |
-
bias=False)
|
33 |
-
self.bn2 = nn.BatchNorm2d(planes)
|
34 |
-
self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
|
35 |
-
self.bn3 = nn.BatchNorm2d(planes * 4)
|
36 |
-
self.relu = nn.ReLU(inplace=True)
|
37 |
-
self.downsample = downsample
|
38 |
-
self.stride = stride
|
39 |
-
|
40 |
-
def forward(self, x):
|
41 |
-
residual = x
|
42 |
-
|
43 |
-
out = self.conv1(x)
|
44 |
-
out = self.bn1(out)
|
45 |
-
out = self.relu(out)
|
46 |
-
|
47 |
-
out = self.conv2(out)
|
48 |
-
out = self.bn2(out)
|
49 |
-
out = self.relu(out)
|
50 |
-
|
51 |
-
out = self.conv3(out)
|
52 |
-
out = self.bn3(out)
|
53 |
-
|
54 |
-
if self.downsample is not None:
|
55 |
-
residual = self.downsample(x)
|
56 |
-
|
57 |
-
out += residual
|
58 |
-
out = self.relu(out)
|
59 |
-
|
60 |
-
return out
|
61 |
-
|
62 |
-
|
63 |
-
class ResNet_Backbone(nn.Module):
|
64 |
-
""" Feature Extrator with ResNet backbone
|
65 |
-
"""
|
66 |
-
|
67 |
-
def __init__(self, model='res50', pretrained=True):
|
68 |
-
if model == 'res50':
|
69 |
-
block, layers = Bottleneck, [3, 4, 6, 3]
|
70 |
-
else:
|
71 |
-
pass # TODO
|
72 |
-
|
73 |
-
self.inplanes = 64
|
74 |
-
super().__init__()
|
75 |
-
npose = 24 * 6
|
76 |
-
self.conv1 = nn.Conv2d(3,
|
77 |
-
64,
|
78 |
-
kernel_size=7,
|
79 |
-
stride=2,
|
80 |
-
padding=3,
|
81 |
-
bias=False)
|
82 |
-
self.bn1 = nn.BatchNorm2d(64)
|
83 |
-
self.relu = nn.ReLU(inplace=True)
|
84 |
-
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
|
85 |
-
self.layer1 = self._make_layer(block, 64, layers[0])
|
86 |
-
self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
|
87 |
-
self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
|
88 |
-
self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
|
89 |
-
self.avgpool = nn.AvgPool2d(7, stride=1)
|
90 |
-
|
91 |
-
if pretrained:
|
92 |
-
resnet_imagenet = resnet.resnet50(pretrained=True)
|
93 |
-
self.load_state_dict(resnet_imagenet.state_dict(), strict=False)
|
94 |
-
logger.info('loaded resnet50 imagenet pretrained model')
|
95 |
-
|
96 |
-
def _make_layer(self, block, planes, blocks, stride=1):
|
97 |
-
downsample = None
|
98 |
-
if stride != 1 or self.inplanes != planes * block.expansion:
|
99 |
-
downsample = nn.Sequential(
|
100 |
-
nn.Conv2d(self.inplanes,
|
101 |
-
planes * block.expansion,
|
102 |
-
kernel_size=1,
|
103 |
-
stride=stride,
|
104 |
-
bias=False),
|
105 |
-
nn.BatchNorm2d(planes * block.expansion),
|
106 |
-
)
|
107 |
-
|
108 |
-
layers = []
|
109 |
-
layers.append(block(self.inplanes, planes, stride, downsample))
|
110 |
-
self.inplanes = planes * block.expansion
|
111 |
-
for i in range(1, blocks):
|
112 |
-
layers.append(block(self.inplanes, planes))
|
113 |
-
|
114 |
-
return nn.Sequential(*layers)
|
115 |
-
|
116 |
-
def _make_deconv_layer(self, num_layers, num_filters, num_kernels):
|
117 |
-
assert num_layers == len(num_filters), \
|
118 |
-
'ERROR: num_deconv_layers is different len(num_deconv_filters)'
|
119 |
-
assert num_layers == len(num_kernels), \
|
120 |
-
'ERROR: num_deconv_layers is different len(num_deconv_filters)'
|
121 |
-
|
122 |
-
def _get_deconv_cfg(deconv_kernel, index):
|
123 |
-
if deconv_kernel == 4:
|
124 |
-
padding = 1
|
125 |
-
output_padding = 0
|
126 |
-
elif deconv_kernel == 3:
|
127 |
-
padding = 1
|
128 |
-
output_padding = 1
|
129 |
-
elif deconv_kernel == 2:
|
130 |
-
padding = 0
|
131 |
-
output_padding = 0
|
132 |
-
|
133 |
-
return deconv_kernel, padding, output_padding
|
134 |
-
|
135 |
-
layers = []
|
136 |
-
for i in range(num_layers):
|
137 |
-
kernel, padding, output_padding = _get_deconv_cfg(
|
138 |
-
num_kernels[i], i)
|
139 |
-
|
140 |
-
planes = num_filters[i]
|
141 |
-
layers.append(
|
142 |
-
nn.ConvTranspose2d(in_channels=self.inplanes,
|
143 |
-
out_channels=planes,
|
144 |
-
kernel_size=kernel,
|
145 |
-
stride=2,
|
146 |
-
padding=padding,
|
147 |
-
output_padding=output_padding,
|
148 |
-
bias=self.deconv_with_bias))
|
149 |
-
layers.append(nn.BatchNorm2d(planes, momentum=BN_MOMENTUM))
|
150 |
-
layers.append(nn.ReLU(inplace=True))
|
151 |
-
self.inplanes = planes
|
152 |
-
|
153 |
-
return nn.Sequential(*layers)
|
154 |
-
|
155 |
-
def forward(self, x):
|
156 |
-
|
157 |
-
batch_size = x.shape[0]
|
158 |
-
|
159 |
-
x = self.conv1(x)
|
160 |
-
x = self.bn1(x)
|
161 |
-
x = self.relu(x)
|
162 |
-
x = self.maxpool(x)
|
163 |
-
|
164 |
-
x1 = self.layer1(x)
|
165 |
-
x2 = self.layer2(x1)
|
166 |
-
x3 = self.layer3(x2)
|
167 |
-
x4 = self.layer4(x3)
|
168 |
-
|
169 |
-
xf = self.avgpool(x4)
|
170 |
-
xf = xf.view(xf.size(0), -1)
|
171 |
-
|
172 |
-
x_featmap = x4
|
173 |
-
|
174 |
-
return x_featmap, xf
|
175 |
-
|
176 |
-
|
177 |
-
class HMR(nn.Module):
|
178 |
-
""" SMPL Iterative Regressor with ResNet50 backbone
|
179 |
-
"""
|
180 |
-
|
181 |
-
def __init__(self, block, layers, smpl_mean_params):
|
182 |
-
self.inplanes = 64
|
183 |
-
super().__init__()
|
184 |
-
npose = 24 * 6
|
185 |
-
self.conv1 = nn.Conv2d(3,
|
186 |
-
64,
|
187 |
-
kernel_size=7,
|
188 |
-
stride=2,
|
189 |
-
padding=3,
|
190 |
-
bias=False)
|
191 |
-
self.bn1 = nn.BatchNorm2d(64)
|
192 |
-
self.relu = nn.ReLU(inplace=True)
|
193 |
-
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
|
194 |
-
self.layer1 = self._make_layer(block, 64, layers[0])
|
195 |
-
self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
|
196 |
-
self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
|
197 |
-
self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
|
198 |
-
self.avgpool = nn.AvgPool2d(7, stride=1)
|
199 |
-
self.fc1 = nn.Linear(512 * block.expansion + npose + 13, 1024)
|
200 |
-
self.drop1 = nn.Dropout()
|
201 |
-
self.fc2 = nn.Linear(1024, 1024)
|
202 |
-
self.drop2 = nn.Dropout()
|
203 |
-
self.decpose = nn.Linear(1024, npose)
|
204 |
-
self.decshape = nn.Linear(1024, 10)
|
205 |
-
self.deccam = nn.Linear(1024, 3)
|
206 |
-
nn.init.xavier_uniform_(self.decpose.weight, gain=0.01)
|
207 |
-
nn.init.xavier_uniform_(self.decshape.weight, gain=0.01)
|
208 |
-
nn.init.xavier_uniform_(self.deccam.weight, gain=0.01)
|
209 |
-
|
210 |
-
for m in self.modules():
|
211 |
-
if isinstance(m, nn.Conv2d):
|
212 |
-
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
|
213 |
-
m.weight.data.normal_(0, math.sqrt(2. / n))
|
214 |
-
elif isinstance(m, nn.BatchNorm2d):
|
215 |
-
m.weight.data.fill_(1)
|
216 |
-
m.bias.data.zero_()
|
217 |
-
|
218 |
-
mean_params = np.load(smpl_mean_params)
|
219 |
-
init_pose = torch.from_numpy(mean_params['pose'][:]).unsqueeze(0)
|
220 |
-
init_shape = torch.from_numpy(
|
221 |
-
mean_params['shape'][:].astype('float32')).unsqueeze(0)
|
222 |
-
init_cam = torch.from_numpy(mean_params['cam']).unsqueeze(0)
|
223 |
-
self.register_buffer('init_pose', init_pose)
|
224 |
-
self.register_buffer('init_shape', init_shape)
|
225 |
-
self.register_buffer('init_cam', init_cam)
|
226 |
-
|
227 |
-
def _make_layer(self, block, planes, blocks, stride=1):
|
228 |
-
downsample = None
|
229 |
-
if stride != 1 or self.inplanes != planes * block.expansion:
|
230 |
-
downsample = nn.Sequential(
|
231 |
-
nn.Conv2d(self.inplanes,
|
232 |
-
planes * block.expansion,
|
233 |
-
kernel_size=1,
|
234 |
-
stride=stride,
|
235 |
-
bias=False),
|
236 |
-
nn.BatchNorm2d(planes * block.expansion),
|
237 |
-
)
|
238 |
-
|
239 |
-
layers = []
|
240 |
-
layers.append(block(self.inplanes, planes, stride, downsample))
|
241 |
-
self.inplanes = planes * block.expansion
|
242 |
-
for i in range(1, blocks):
|
243 |
-
layers.append(block(self.inplanes, planes))
|
244 |
-
|
245 |
-
return nn.Sequential(*layers)
|
246 |
-
|
247 |
-
def forward(self,
|
248 |
-
x,
|
249 |
-
init_pose=None,
|
250 |
-
init_shape=None,
|
251 |
-
init_cam=None,
|
252 |
-
n_iter=3):
|
253 |
-
|
254 |
-
batch_size = x.shape[0]
|
255 |
-
|
256 |
-
if init_pose is None:
|
257 |
-
init_pose = self.init_pose.expand(batch_size, -1)
|
258 |
-
if init_shape is None:
|
259 |
-
init_shape = self.init_shape.expand(batch_size, -1)
|
260 |
-
if init_cam is None:
|
261 |
-
init_cam = self.init_cam.expand(batch_size, -1)
|
262 |
-
|
263 |
-
x = self.conv1(x)
|
264 |
-
x = self.bn1(x)
|
265 |
-
x = self.relu(x)
|
266 |
-
x = self.maxpool(x)
|
267 |
-
|
268 |
-
x1 = self.layer1(x)
|
269 |
-
x2 = self.layer2(x1)
|
270 |
-
x3 = self.layer3(x2)
|
271 |
-
x4 = self.layer4(x3)
|
272 |
-
|
273 |
-
xf = self.avgpool(x4)
|
274 |
-
xf = xf.view(xf.size(0), -1)
|
275 |
-
|
276 |
-
pred_pose = init_pose
|
277 |
-
pred_shape = init_shape
|
278 |
-
pred_cam = init_cam
|
279 |
-
for i in range(n_iter):
|
280 |
-
xc = torch.cat([xf, pred_pose, pred_shape, pred_cam], 1)
|
281 |
-
xc = self.fc1(xc)
|
282 |
-
xc = self.drop1(xc)
|
283 |
-
xc = self.fc2(xc)
|
284 |
-
xc = self.drop2(xc)
|
285 |
-
pred_pose = self.decpose(xc) + pred_pose
|
286 |
-
pred_shape = self.decshape(xc) + pred_shape
|
287 |
-
pred_cam = self.deccam(xc) + pred_cam
|
288 |
-
|
289 |
-
pred_rotmat = rot6d_to_rotmat(pred_pose).view(batch_size, 24, 3, 3)
|
290 |
-
|
291 |
-
return pred_rotmat, pred_shape, pred_cam
|
292 |
-
|
293 |
-
|
294 |
-
def hmr(smpl_mean_params, pretrained=True, **kwargs):
|
295 |
-
""" Constructs an HMR model with ResNet50 backbone.
|
296 |
-
Args:
|
297 |
-
pretrained (bool): If True, returns a model pre-trained on ImageNet
|
298 |
-
"""
|
299 |
-
model = HMR(Bottleneck, [3, 4, 6, 3], smpl_mean_params, **kwargs)
|
300 |
-
if pretrained:
|
301 |
-
resnet_imagenet = resnet.resnet50(pretrained=True)
|
302 |
-
model.load_state_dict(resnet_imagenet.state_dict(), strict=False)
|
303 |
-
return model
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /models /maf_extractor.py
DELETED
@@ -1,135 +0,0 @@
|
|
1 |
-
# This script is borrowed and extended from https://github.com/shunsukesaito/PIFu/blob/master/lib/model/SurfaceClassifier.py
|
2 |
-
|
3 |
-
from packaging import version
|
4 |
-
import torch
|
5 |
-
import scipy
|
6 |
-
import numpy as np
|
7 |
-
import torch.nn as nn
|
8 |
-
import torch.nn.functional as F
|
9 |
-
|
10 |
-
from lib.common.config import cfg
|
11 |
-
from lib.pymaf.utils.geometry import projection
|
12 |
-
from lib.pymaf.core.path_config import MESH_DOWNSAMPLEING
|
13 |
-
|
14 |
-
import logging
|
15 |
-
|
16 |
-
logger = logging.getLogger(__name__)
|
17 |
-
|
18 |
-
|
19 |
-
class MAF_Extractor(nn.Module):
|
20 |
-
''' Mesh-aligned Feature Extrator
|
21 |
-
As discussed in the paper, we extract mesh-aligned features based on 2D projection of the mesh vertices.
|
22 |
-
The features extrated from spatial feature maps will go through a MLP for dimension reduction.
|
23 |
-
'''
|
24 |
-
|
25 |
-
def __init__(self, device=torch.device('cuda')):
|
26 |
-
super().__init__()
|
27 |
-
|
28 |
-
self.device = device
|
29 |
-
self.filters = []
|
30 |
-
self.num_views = 1
|
31 |
-
filter_channels = cfg.MODEL.PyMAF.MLP_DIM
|
32 |
-
self.last_op = nn.ReLU(True)
|
33 |
-
|
34 |
-
for l in range(0, len(filter_channels) - 1):
|
35 |
-
if 0 != l:
|
36 |
-
self.filters.append(
|
37 |
-
nn.Conv1d(filter_channels[l] + filter_channels[0],
|
38 |
-
filter_channels[l + 1], 1))
|
39 |
-
else:
|
40 |
-
self.filters.append(
|
41 |
-
nn.Conv1d(filter_channels[l], filter_channels[l + 1], 1))
|
42 |
-
|
43 |
-
self.add_module("conv%d" % l, self.filters[l])
|
44 |
-
|
45 |
-
self.im_feat = None
|
46 |
-
self.cam = None
|
47 |
-
|
48 |
-
# downsample SMPL mesh and assign part labels
|
49 |
-
# from https://github.com/nkolot/GraphCMR/blob/master/data/mesh_downsampling.npz
|
50 |
-
smpl_mesh_graph = np.load(MESH_DOWNSAMPLEING,
|
51 |
-
allow_pickle=True,
|
52 |
-
encoding='latin1')
|
53 |
-
|
54 |
-
A = smpl_mesh_graph['A']
|
55 |
-
U = smpl_mesh_graph['U']
|
56 |
-
D = smpl_mesh_graph['D'] # shape: (2,)
|
57 |
-
|
58 |
-
# downsampling
|
59 |
-
ptD = []
|
60 |
-
for i in range(len(D)):
|
61 |
-
d = scipy.sparse.coo_matrix(D[i])
|
62 |
-
i = torch.LongTensor(np.array([d.row, d.col]))
|
63 |
-
v = torch.FloatTensor(d.data)
|
64 |
-
ptD.append(torch.sparse.FloatTensor(i, v, d.shape))
|
65 |
-
|
66 |
-
# downsampling mapping from 6890 points to 431 points
|
67 |
-
# ptD[0].to_dense() - Size: [1723, 6890]
|
68 |
-
# ptD[1].to_dense() - Size: [431. 1723]
|
69 |
-
Dmap = torch.matmul(ptD[1].to_dense(),
|
70 |
-
ptD[0].to_dense()) # 6890 -> 431
|
71 |
-
self.register_buffer('Dmap', Dmap)
|
72 |
-
|
73 |
-
def reduce_dim(self, feature):
|
74 |
-
'''
|
75 |
-
Dimension reduction by multi-layer perceptrons
|
76 |
-
:param feature: list of [B, C_s, N] point-wise features before dimension reduction
|
77 |
-
:return: [B, C_p x N] concatantion of point-wise features after dimension reduction
|
78 |
-
'''
|
79 |
-
y = feature
|
80 |
-
tmpy = feature
|
81 |
-
for i, f in enumerate(self.filters):
|
82 |
-
y = self._modules['conv' +
|
83 |
-
str(i)](y if i == 0 else torch.cat([y, tmpy], 1))
|
84 |
-
if i != len(self.filters) - 1:
|
85 |
-
y = F.leaky_relu(y)
|
86 |
-
if self.num_views > 1 and i == len(self.filters) // 2:
|
87 |
-
y = y.view(-1, self.num_views, y.shape[1],
|
88 |
-
y.shape[2]).mean(dim=1)
|
89 |
-
tmpy = feature.view(-1, self.num_views, feature.shape[1],
|
90 |
-
feature.shape[2]).mean(dim=1)
|
91 |
-
|
92 |
-
y = self.last_op(y)
|
93 |
-
|
94 |
-
y = y.view(y.shape[0], -1)
|
95 |
-
return y
|
96 |
-
|
97 |
-
def sampling(self, points, im_feat=None, z_feat=None):
|
98 |
-
'''
|
99 |
-
Given 2D points, sample the point-wise features for each point,
|
100 |
-
the dimension of point-wise features will be reduced from C_s to C_p by MLP.
|
101 |
-
Image features should be pre-computed before this call.
|
102 |
-
:param points: [B, N, 2] image coordinates of points
|
103 |
-
:im_feat: [B, C_s, H_s, W_s] spatial feature maps
|
104 |
-
:return: [B, C_p x N] concatantion of point-wise features after dimension reduction
|
105 |
-
'''
|
106 |
-
if im_feat is None:
|
107 |
-
im_feat = self.im_feat
|
108 |
-
|
109 |
-
batch_size = im_feat.shape[0]
|
110 |
-
|
111 |
-
if version.parse(torch.__version__) >= version.parse('1.3.0'):
|
112 |
-
# Default grid_sample behavior has changed to align_corners=False since 1.3.0.
|
113 |
-
point_feat = torch.nn.functional.grid_sample(
|
114 |
-
im_feat, points.unsqueeze(2), align_corners=True)[..., 0]
|
115 |
-
else:
|
116 |
-
point_feat = torch.nn.functional.grid_sample(
|
117 |
-
im_feat, points.unsqueeze(2))[..., 0]
|
118 |
-
|
119 |
-
mesh_align_feat = self.reduce_dim(point_feat)
|
120 |
-
return mesh_align_feat
|
121 |
-
|
122 |
-
def forward(self, p, s_feat=None, cam=None, **kwargs):
|
123 |
-
''' Returns mesh-aligned features for the 3D mesh points.
|
124 |
-
Args:
|
125 |
-
p (tensor): [B, N_m, 3] mesh vertices
|
126 |
-
s_feat (tensor): [B, C_s, H_s, W_s] spatial feature maps
|
127 |
-
cam (tensor): [B, 3] camera
|
128 |
-
Return:
|
129 |
-
mesh_align_feat (tensor): [B, C_p x N_m] mesh-aligned features
|
130 |
-
'''
|
131 |
-
if cam is None:
|
132 |
-
cam = self.cam
|
133 |
-
p_proj_2d = projection(p, cam, retain_z=False)
|
134 |
-
mesh_align_feat = self.sampling(p_proj_2d, s_feat)
|
135 |
-
return mesh_align_feat
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /models /res_module.py
DELETED
@@ -1,385 +0,0 @@
|
|
1 |
-
# code brought in part from https://github.com/microsoft/human-pose-estimation.pytorch/blob/master/lib/models/pose_resnet.py
|
2 |
-
|
3 |
-
from __future__ import absolute_import
|
4 |
-
from __future__ import division
|
5 |
-
from __future__ import print_function
|
6 |
-
|
7 |
-
import torch
|
8 |
-
import torch.nn as nn
|
9 |
-
import torch.nn.functional as F
|
10 |
-
from collections import OrderedDict
|
11 |
-
import os
|
12 |
-
from lib.pymaf.core.cfgs import cfg
|
13 |
-
|
14 |
-
import logging
|
15 |
-
|
16 |
-
logger = logging.getLogger(__name__)
|
17 |
-
|
18 |
-
BN_MOMENTUM = 0.1
|
19 |
-
|
20 |
-
|
21 |
-
def conv3x3(in_planes, out_planes, stride=1, bias=False, groups=1):
|
22 |
-
"""3x3 convolution with padding"""
|
23 |
-
return nn.Conv2d(in_planes * groups,
|
24 |
-
out_planes * groups,
|
25 |
-
kernel_size=3,
|
26 |
-
stride=stride,
|
27 |
-
padding=1,
|
28 |
-
bias=bias,
|
29 |
-
groups=groups)
|
30 |
-
|
31 |
-
|
32 |
-
class BasicBlock(nn.Module):
|
33 |
-
expansion = 1
|
34 |
-
|
35 |
-
def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1):
|
36 |
-
super().__init__()
|
37 |
-
self.conv1 = conv3x3(inplanes, planes, stride, groups=groups)
|
38 |
-
self.bn1 = nn.BatchNorm2d(planes * groups, momentum=BN_MOMENTUM)
|
39 |
-
self.relu = nn.ReLU(inplace=True)
|
40 |
-
self.conv2 = conv3x3(planes, planes, groups=groups)
|
41 |
-
self.bn2 = nn.BatchNorm2d(planes * groups, momentum=BN_MOMENTUM)
|
42 |
-
self.downsample = downsample
|
43 |
-
self.stride = stride
|
44 |
-
|
45 |
-
def forward(self, x):
|
46 |
-
residual = x
|
47 |
-
|
48 |
-
out = self.conv1(x)
|
49 |
-
out = self.bn1(out)
|
50 |
-
out = self.relu(out)
|
51 |
-
|
52 |
-
out = self.conv2(out)
|
53 |
-
out = self.bn2(out)
|
54 |
-
|
55 |
-
if self.downsample is not None:
|
56 |
-
residual = self.downsample(x)
|
57 |
-
|
58 |
-
out += residual
|
59 |
-
out = self.relu(out)
|
60 |
-
|
61 |
-
return out
|
62 |
-
|
63 |
-
|
64 |
-
class Bottleneck(nn.Module):
|
65 |
-
expansion = 4
|
66 |
-
|
67 |
-
def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1):
|
68 |
-
super().__init__()
|
69 |
-
self.conv1 = nn.Conv2d(inplanes * groups,
|
70 |
-
planes * groups,
|
71 |
-
kernel_size=1,
|
72 |
-
bias=False,
|
73 |
-
groups=groups)
|
74 |
-
self.bn1 = nn.BatchNorm2d(planes * groups, momentum=BN_MOMENTUM)
|
75 |
-
self.conv2 = nn.Conv2d(planes * groups,
|
76 |
-
planes * groups,
|
77 |
-
kernel_size=3,
|
78 |
-
stride=stride,
|
79 |
-
padding=1,
|
80 |
-
bias=False,
|
81 |
-
groups=groups)
|
82 |
-
self.bn2 = nn.BatchNorm2d(planes * groups, momentum=BN_MOMENTUM)
|
83 |
-
self.conv3 = nn.Conv2d(planes * groups,
|
84 |
-
planes * self.expansion * groups,
|
85 |
-
kernel_size=1,
|
86 |
-
bias=False,
|
87 |
-
groups=groups)
|
88 |
-
self.bn3 = nn.BatchNorm2d(planes * self.expansion * groups,
|
89 |
-
momentum=BN_MOMENTUM)
|
90 |
-
self.relu = nn.ReLU(inplace=True)
|
91 |
-
self.downsample = downsample
|
92 |
-
self.stride = stride
|
93 |
-
|
94 |
-
def forward(self, x):
|
95 |
-
residual = x
|
96 |
-
|
97 |
-
out = self.conv1(x)
|
98 |
-
out = self.bn1(out)
|
99 |
-
out = self.relu(out)
|
100 |
-
|
101 |
-
out = self.conv2(out)
|
102 |
-
out = self.bn2(out)
|
103 |
-
out = self.relu(out)
|
104 |
-
|
105 |
-
out = self.conv3(out)
|
106 |
-
out = self.bn3(out)
|
107 |
-
|
108 |
-
if self.downsample is not None:
|
109 |
-
residual = self.downsample(x)
|
110 |
-
|
111 |
-
out += residual
|
112 |
-
out = self.relu(out)
|
113 |
-
|
114 |
-
return out
|
115 |
-
|
116 |
-
|
117 |
-
resnet_spec = {
|
118 |
-
18: (BasicBlock, [2, 2, 2, 2]),
|
119 |
-
34: (BasicBlock, [3, 4, 6, 3]),
|
120 |
-
50: (Bottleneck, [3, 4, 6, 3]),
|
121 |
-
101: (Bottleneck, [3, 4, 23, 3]),
|
122 |
-
152: (Bottleneck, [3, 8, 36, 3])
|
123 |
-
}
|
124 |
-
|
125 |
-
|
126 |
-
class IUV_predict_layer(nn.Module):
|
127 |
-
def __init__(self,
|
128 |
-
feat_dim=256,
|
129 |
-
final_cov_k=3,
|
130 |
-
part_out_dim=25,
|
131 |
-
with_uv=True):
|
132 |
-
super().__init__()
|
133 |
-
|
134 |
-
self.with_uv = with_uv
|
135 |
-
if self.with_uv:
|
136 |
-
self.predict_u = nn.Conv2d(in_channels=feat_dim,
|
137 |
-
out_channels=25,
|
138 |
-
kernel_size=final_cov_k,
|
139 |
-
stride=1,
|
140 |
-
padding=1 if final_cov_k == 3 else 0)
|
141 |
-
|
142 |
-
self.predict_v = nn.Conv2d(in_channels=feat_dim,
|
143 |
-
out_channels=25,
|
144 |
-
kernel_size=final_cov_k,
|
145 |
-
stride=1,
|
146 |
-
padding=1 if final_cov_k == 3 else 0)
|
147 |
-
|
148 |
-
self.predict_ann_index = nn.Conv2d(
|
149 |
-
in_channels=feat_dim,
|
150 |
-
out_channels=15,
|
151 |
-
kernel_size=final_cov_k,
|
152 |
-
stride=1,
|
153 |
-
padding=1 if final_cov_k == 3 else 0)
|
154 |
-
|
155 |
-
self.predict_uv_index = nn.Conv2d(in_channels=feat_dim,
|
156 |
-
out_channels=25,
|
157 |
-
kernel_size=final_cov_k,
|
158 |
-
stride=1,
|
159 |
-
padding=1 if final_cov_k == 3 else 0)
|
160 |
-
|
161 |
-
self.inplanes = feat_dim
|
162 |
-
|
163 |
-
def _make_layer(self, block, planes, blocks, stride=1):
|
164 |
-
downsample = None
|
165 |
-
if stride != 1 or self.inplanes != planes * block.expansion:
|
166 |
-
downsample = nn.Sequential(
|
167 |
-
nn.Conv2d(self.inplanes,
|
168 |
-
planes * block.expansion,
|
169 |
-
kernel_size=1,
|
170 |
-
stride=stride,
|
171 |
-
bias=False),
|
172 |
-
nn.BatchNorm2d(planes * block.expansion),
|
173 |
-
)
|
174 |
-
|
175 |
-
layers = []
|
176 |
-
layers.append(block(self.inplanes, planes, stride, downsample))
|
177 |
-
self.inplanes = planes * block.expansion
|
178 |
-
for i in range(1, blocks):
|
179 |
-
layers.append(block(self.inplanes, planes))
|
180 |
-
|
181 |
-
return nn.Sequential(*layers)
|
182 |
-
|
183 |
-
def forward(self, x):
|
184 |
-
return_dict = {}
|
185 |
-
|
186 |
-
predict_uv_index = self.predict_uv_index(x)
|
187 |
-
predict_ann_index = self.predict_ann_index(x)
|
188 |
-
|
189 |
-
return_dict['predict_uv_index'] = predict_uv_index
|
190 |
-
return_dict['predict_ann_index'] = predict_ann_index
|
191 |
-
|
192 |
-
if self.with_uv:
|
193 |
-
predict_u = self.predict_u(x)
|
194 |
-
predict_v = self.predict_v(x)
|
195 |
-
return_dict['predict_u'] = predict_u
|
196 |
-
return_dict['predict_v'] = predict_v
|
197 |
-
else:
|
198 |
-
return_dict['predict_u'] = None
|
199 |
-
return_dict['predict_v'] = None
|
200 |
-
# return_dict['predict_u'] = torch.zeros(predict_uv_index.shape).to(predict_uv_index.device)
|
201 |
-
# return_dict['predict_v'] = torch.zeros(predict_uv_index.shape).to(predict_uv_index.device)
|
202 |
-
|
203 |
-
return return_dict
|
204 |
-
|
205 |
-
|
206 |
-
class SmplResNet(nn.Module):
|
207 |
-
def __init__(self,
|
208 |
-
resnet_nums,
|
209 |
-
in_channels=3,
|
210 |
-
num_classes=229,
|
211 |
-
last_stride=2,
|
212 |
-
n_extra_feat=0,
|
213 |
-
truncate=0,
|
214 |
-
**kwargs):
|
215 |
-
super().__init__()
|
216 |
-
|
217 |
-
self.inplanes = 64
|
218 |
-
self.truncate = truncate
|
219 |
-
# extra = cfg.MODEL.EXTRA
|
220 |
-
# self.deconv_with_bias = extra.DECONV_WITH_BIAS
|
221 |
-
block, layers = resnet_spec[resnet_nums]
|
222 |
-
|
223 |
-
self.conv1 = nn.Conv2d(in_channels,
|
224 |
-
64,
|
225 |
-
kernel_size=7,
|
226 |
-
stride=2,
|
227 |
-
padding=3,
|
228 |
-
bias=False)
|
229 |
-
self.bn1 = nn.BatchNorm2d(64, momentum=BN_MOMENTUM)
|
230 |
-
self.relu = nn.ReLU(inplace=True)
|
231 |
-
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
|
232 |
-
self.layer1 = self._make_layer(block, 64, layers[0])
|
233 |
-
self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
|
234 |
-
self.layer3 = self._make_layer(block, 256, layers[2],
|
235 |
-
stride=2) if truncate < 2 else None
|
236 |
-
self.layer4 = self._make_layer(
|
237 |
-
block, 512, layers[3],
|
238 |
-
stride=last_stride) if truncate < 1 else None
|
239 |
-
|
240 |
-
self.avg_pooling = nn.AdaptiveAvgPool2d(1)
|
241 |
-
|
242 |
-
self.num_classes = num_classes
|
243 |
-
if num_classes > 0:
|
244 |
-
self.final_layer = nn.Linear(512 * block.expansion, num_classes)
|
245 |
-
nn.init.xavier_uniform_(self.final_layer.weight, gain=0.01)
|
246 |
-
|
247 |
-
self.n_extra_feat = n_extra_feat
|
248 |
-
if n_extra_feat > 0:
|
249 |
-
self.trans_conv = nn.Sequential(
|
250 |
-
nn.Conv2d(n_extra_feat + 512 * block.expansion,
|
251 |
-
512 * block.expansion,
|
252 |
-
kernel_size=1,
|
253 |
-
bias=False),
|
254 |
-
nn.BatchNorm2d(512 * block.expansion, momentum=BN_MOMENTUM),
|
255 |
-
nn.ReLU(True))
|
256 |
-
|
257 |
-
def _make_layer(self, block, planes, blocks, stride=1):
|
258 |
-
downsample = None
|
259 |
-
if stride != 1 or self.inplanes != planes * block.expansion:
|
260 |
-
downsample = nn.Sequential(
|
261 |
-
nn.Conv2d(self.inplanes,
|
262 |
-
planes * block.expansion,
|
263 |
-
kernel_size=1,
|
264 |
-
stride=stride,
|
265 |
-
bias=False),
|
266 |
-
nn.BatchNorm2d(planes * block.expansion, momentum=BN_MOMENTUM),
|
267 |
-
)
|
268 |
-
|
269 |
-
layers = []
|
270 |
-
layers.append(block(self.inplanes, planes, stride, downsample))
|
271 |
-
self.inplanes = planes * block.expansion
|
272 |
-
for i in range(1, blocks):
|
273 |
-
layers.append(block(self.inplanes, planes))
|
274 |
-
|
275 |
-
return nn.Sequential(*layers)
|
276 |
-
|
277 |
-
def forward(self, x, infeat=None):
|
278 |
-
x = self.conv1(x)
|
279 |
-
x = self.bn1(x)
|
280 |
-
x = self.relu(x)
|
281 |
-
x = self.maxpool(x)
|
282 |
-
|
283 |
-
x1 = self.layer1(x)
|
284 |
-
x2 = self.layer2(x1)
|
285 |
-
x3 = self.layer3(x2) if self.truncate < 2 else x2
|
286 |
-
x4 = self.layer4(x3) if self.truncate < 1 else x3
|
287 |
-
|
288 |
-
if infeat is not None:
|
289 |
-
x4 = self.trans_conv(torch.cat([infeat, x4], 1))
|
290 |
-
|
291 |
-
if self.num_classes > 0:
|
292 |
-
xp = self.avg_pooling(x4)
|
293 |
-
cls = self.final_layer(xp.view(xp.size(0), -1))
|
294 |
-
if not cfg.DANET.USE_MEAN_PARA:
|
295 |
-
# for non-negative scale
|
296 |
-
scale = F.relu(cls[:, 0]).unsqueeze(1)
|
297 |
-
cls = torch.cat((scale, cls[:, 1:]), dim=1)
|
298 |
-
else:
|
299 |
-
cls = None
|
300 |
-
|
301 |
-
return cls, {'x4': x4}
|
302 |
-
|
303 |
-
def init_weights(self, pretrained=''):
|
304 |
-
if os.path.isfile(pretrained):
|
305 |
-
logger.info('=> loading pretrained model {}'.format(pretrained))
|
306 |
-
# self.load_state_dict(pretrained_state_dict, strict=False)
|
307 |
-
checkpoint = torch.load(pretrained)
|
308 |
-
if isinstance(checkpoint, OrderedDict):
|
309 |
-
# state_dict = checkpoint
|
310 |
-
state_dict_old = self.state_dict()
|
311 |
-
for key in state_dict_old.keys():
|
312 |
-
if key in checkpoint.keys():
|
313 |
-
if state_dict_old[key].shape != checkpoint[key].shape:
|
314 |
-
del checkpoint[key]
|
315 |
-
state_dict = checkpoint
|
316 |
-
elif isinstance(checkpoint, dict) and 'state_dict' in checkpoint:
|
317 |
-
state_dict_old = checkpoint['state_dict']
|
318 |
-
state_dict = OrderedDict()
|
319 |
-
# delete 'module.' because it is saved from DataParallel module
|
320 |
-
for key in state_dict_old.keys():
|
321 |
-
if key.startswith('module.'):
|
322 |
-
# state_dict[key[7:]] = state_dict[key]
|
323 |
-
# state_dict.pop(key)
|
324 |
-
state_dict[key[7:]] = state_dict_old[key]
|
325 |
-
else:
|
326 |
-
state_dict[key] = state_dict_old[key]
|
327 |
-
else:
|
328 |
-
raise RuntimeError(
|
329 |
-
'No state_dict found in checkpoint file {}'.format(
|
330 |
-
pretrained))
|
331 |
-
self.load_state_dict(state_dict, strict=False)
|
332 |
-
else:
|
333 |
-
logger.error('=> imagenet pretrained model dose not exist')
|
334 |
-
logger.error('=> please download it first')
|
335 |
-
raise ValueError('imagenet pretrained model does not exist')
|
336 |
-
|
337 |
-
|
338 |
-
class LimbResLayers(nn.Module):
|
339 |
-
def __init__(self,
|
340 |
-
resnet_nums,
|
341 |
-
inplanes,
|
342 |
-
outplanes=None,
|
343 |
-
groups=1,
|
344 |
-
**kwargs):
|
345 |
-
super().__init__()
|
346 |
-
|
347 |
-
self.inplanes = inplanes
|
348 |
-
block, layers = resnet_spec[resnet_nums]
|
349 |
-
self.outplanes = 512 if outplanes == None else outplanes
|
350 |
-
self.layer4 = self._make_layer(block,
|
351 |
-
self.outplanes,
|
352 |
-
layers[3],
|
353 |
-
stride=2,
|
354 |
-
groups=groups)
|
355 |
-
|
356 |
-
self.avg_pooling = nn.AdaptiveAvgPool2d(1)
|
357 |
-
|
358 |
-
def _make_layer(self, block, planes, blocks, stride=1, groups=1):
|
359 |
-
downsample = None
|
360 |
-
if stride != 1 or self.inplanes != planes * block.expansion:
|
361 |
-
downsample = nn.Sequential(
|
362 |
-
nn.Conv2d(self.inplanes * groups,
|
363 |
-
planes * block.expansion * groups,
|
364 |
-
kernel_size=1,
|
365 |
-
stride=stride,
|
366 |
-
bias=False,
|
367 |
-
groups=groups),
|
368 |
-
nn.BatchNorm2d(planes * block.expansion * groups,
|
369 |
-
momentum=BN_MOMENTUM),
|
370 |
-
)
|
371 |
-
|
372 |
-
layers = []
|
373 |
-
layers.append(
|
374 |
-
block(self.inplanes, planes, stride, downsample, groups=groups))
|
375 |
-
self.inplanes = planes * block.expansion
|
376 |
-
for i in range(1, blocks):
|
377 |
-
layers.append(block(self.inplanes, planes, groups=groups))
|
378 |
-
|
379 |
-
return nn.Sequential(*layers)
|
380 |
-
|
381 |
-
def forward(self, x):
|
382 |
-
x = self.layer4(x)
|
383 |
-
x = self.avg_pooling(x)
|
384 |
-
|
385 |
-
return x
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /utils / __init__.py
DELETED
File without changes
|
lib / pymaf /utils / geometry.py
DELETED
@@ -1,435 +0,0 @@
|
|
1 |
-
import torch
|
2 |
-
import numpy as np
|
3 |
-
from torch.nn import functional as F
|
4 |
-
"""
|
5 |
-
Useful geometric operations, e.g. Perspective projection and a differentiable Rodrigues formula
|
6 |
-
Parts of the code are taken from https://github.com/MandyMo/pytorch_HMR
|
7 |
-
"""
|
8 |
-
|
9 |
-
|
10 |
-
def batch_rodrigues(theta):
|
11 |
-
"""Convert axis-angle representation to rotation matrix.
|
12 |
-
Args:
|
13 |
-
theta: size = [B, 3]
|
14 |
-
Returns:
|
15 |
-
Rotation matrix corresponding to the quaternion -- size = [B, 3, 3]
|
16 |
-
"""
|
17 |
-
l1norm = torch.norm(theta + 1e-8, p=2, dim=1)
|
18 |
-
angle = torch.unsqueeze(l1norm, -1)
|
19 |
-
normalized = torch.div(theta, angle)
|
20 |
-
angle = angle * 0.5
|
21 |
-
v_cos = torch.cos(angle)
|
22 |
-
v_sin = torch.sin(angle)
|
23 |
-
quat = torch.cat([v_cos, v_sin * normalized], dim=1)
|
24 |
-
return quat_to_rotmat(quat)
|
25 |
-
|
26 |
-
|
27 |
-
def quat_to_rotmat(quat):
|
28 |
-
"""Convert quaternion coefficients to rotation matrix.
|
29 |
-
Args:
|
30 |
-
quat: size = [B, 4] 4 <===>(w, x, y, z)
|
31 |
-
Returns:
|
32 |
-
Rotation matrix corresponding to the quaternion -- size = [B, 3, 3]
|
33 |
-
"""
|
34 |
-
norm_quat = quat
|
35 |
-
norm_quat = norm_quat / norm_quat.norm(p=2, dim=1, keepdim=True)
|
36 |
-
w, x, y, z = norm_quat[:, 0], norm_quat[:, 1], norm_quat[:,
|
37 |
-
2], norm_quat[:,
|
38 |
-
3]
|
39 |
-
|
40 |
-
B = quat.size(0)
|
41 |
-
|
42 |
-
w2, x2, y2, z2 = w.pow(2), x.pow(2), y.pow(2), z.pow(2)
|
43 |
-
wx, wy, wz = w * x, w * y, w * z
|
44 |
-
xy, xz, yz = x * y, x * z, y * z
|
45 |
-
|
46 |
-
rotMat = torch.stack([
|
47 |
-
w2 + x2 - y2 - z2, 2 * xy - 2 * wz, 2 * wy + 2 * xz, 2 * wz + 2 * xy,
|
48 |
-
w2 - x2 + y2 - z2, 2 * yz - 2 * wx, 2 * xz - 2 * wy, 2 * wx + 2 * yz,
|
49 |
-
w2 - x2 - y2 + z2
|
50 |
-
],
|
51 |
-
dim=1).view(B, 3, 3)
|
52 |
-
return rotMat
|
53 |
-
|
54 |
-
|
55 |
-
def rotation_matrix_to_angle_axis(rotation_matrix):
|
56 |
-
"""
|
57 |
-
This function is borrowed from https://github.com/kornia/kornia
|
58 |
-
Convert 3x4 rotation matrix to Rodrigues vector
|
59 |
-
Args:
|
60 |
-
rotation_matrix (Tensor): rotation matrix.
|
61 |
-
Returns:
|
62 |
-
Tensor: Rodrigues vector transformation.
|
63 |
-
Shape:
|
64 |
-
- Input: :math:`(N, 3, 4)`
|
65 |
-
- Output: :math:`(N, 3)`
|
66 |
-
Example:
|
67 |
-
>>> input = torch.rand(2, 3, 4) # Nx4x4
|
68 |
-
>>> output = tgm.rotation_matrix_to_angle_axis(input) # Nx3
|
69 |
-
"""
|
70 |
-
if rotation_matrix.shape[1:] == (3, 3):
|
71 |
-
rot_mat = rotation_matrix.reshape(-1, 3, 3)
|
72 |
-
hom = torch.tensor([0, 0, 1],
|
73 |
-
dtype=torch.float32,
|
74 |
-
device=rotation_matrix.device).reshape(
|
75 |
-
1, 3, 1).expand(rot_mat.shape[0], -1, -1)
|
76 |
-
rotation_matrix = torch.cat([rot_mat, hom], dim=-1)
|
77 |
-
|
78 |
-
quaternion = rotation_matrix_to_quaternion(rotation_matrix)
|
79 |
-
aa = quaternion_to_angle_axis(quaternion)
|
80 |
-
aa[torch.isnan(aa)] = 0.0
|
81 |
-
return aa
|
82 |
-
|
83 |
-
|
84 |
-
def quaternion_to_angle_axis(quaternion: torch.Tensor) -> torch.Tensor:
|
85 |
-
"""
|
86 |
-
This function is borrowed from https://github.com/kornia/kornia
|
87 |
-
Convert quaternion vector to angle axis of rotation.
|
88 |
-
Adapted from ceres C++ library: ceres-solver/include/ceres/rotation.h
|
89 |
-
Args:
|
90 |
-
quaternion (torch.Tensor): tensor with quaternions.
|
91 |
-
Return:
|
92 |
-
torch.Tensor: tensor with angle axis of rotation.
|
93 |
-
Shape:
|
94 |
-
- Input: :math:`(*, 4)` where `*` means, any number of dimensions
|
95 |
-
- Output: :math:`(*, 3)`
|
96 |
-
Example:
|
97 |
-
>>> quaternion = torch.rand(2, 4) # Nx4
|
98 |
-
>>> angle_axis = tgm.quaternion_to_angle_axis(quaternion) # Nx3
|
99 |
-
"""
|
100 |
-
if not torch.is_tensor(quaternion):
|
101 |
-
raise TypeError("Input type is not a torch.Tensor. Got {}".format(
|
102 |
-
type(quaternion)))
|
103 |
-
|
104 |
-
if not quaternion.shape[-1] == 4:
|
105 |
-
raise ValueError(
|
106 |
-
"Input must be a tensor of shape Nx4 or 4. Got {}".format(
|
107 |
-
quaternion.shape))
|
108 |
-
# unpack input and compute conversion
|
109 |
-
q1: torch.Tensor = quaternion[..., 1]
|
110 |
-
q2: torch.Tensor = quaternion[..., 2]
|
111 |
-
q3: torch.Tensor = quaternion[..., 3]
|
112 |
-
sin_squared_theta: torch.Tensor = q1 * q1 + q2 * q2 + q3 * q3
|
113 |
-
|
114 |
-
sin_theta: torch.Tensor = torch.sqrt(sin_squared_theta)
|
115 |
-
cos_theta: torch.Tensor = quaternion[..., 0]
|
116 |
-
two_theta: torch.Tensor = 2.0 * torch.where(
|
117 |
-
cos_theta < 0.0, torch.atan2(-sin_theta, -cos_theta),
|
118 |
-
torch.atan2(sin_theta, cos_theta))
|
119 |
-
|
120 |
-
k_pos: torch.Tensor = two_theta / sin_theta
|
121 |
-
k_neg: torch.Tensor = 2.0 * torch.ones_like(sin_theta)
|
122 |
-
k: torch.Tensor = torch.where(sin_squared_theta > 0.0, k_pos, k_neg)
|
123 |
-
|
124 |
-
angle_axis: torch.Tensor = torch.zeros_like(quaternion)[..., :3]
|
125 |
-
angle_axis[..., 0] += q1 * k
|
126 |
-
angle_axis[..., 1] += q2 * k
|
127 |
-
angle_axis[..., 2] += q3 * k
|
128 |
-
return angle_axis
|
129 |
-
|
130 |
-
|
131 |
-
def rotation_matrix_to_quaternion(rotation_matrix, eps=1e-6):
|
132 |
-
"""
|
133 |
-
This function is borrowed from https://github.com/kornia/kornia
|
134 |
-
Convert 3x4 rotation matrix to 4d quaternion vector
|
135 |
-
This algorithm is based on algorithm described in
|
136 |
-
https://github.com/KieranWynn/pyquaternion/blob/master/pyquaternion/quaternion.py#L201
|
137 |
-
Args:
|
138 |
-
rotation_matrix (Tensor): the rotation matrix to convert.
|
139 |
-
Return:
|
140 |
-
Tensor: the rotation in quaternion
|
141 |
-
Shape:
|
142 |
-
- Input: :math:`(N, 3, 4)`
|
143 |
-
- Output: :math:`(N, 4)`
|
144 |
-
Example:
|
145 |
-
>>> input = torch.rand(4, 3, 4) # Nx3x4
|
146 |
-
>>> output = tgm.rotation_matrix_to_quaternion(input) # Nx4
|
147 |
-
"""
|
148 |
-
if not torch.is_tensor(rotation_matrix):
|
149 |
-
raise TypeError("Input type is not a torch.Tensor. Got {}".format(
|
150 |
-
type(rotation_matrix)))
|
151 |
-
|
152 |
-
if len(rotation_matrix.shape) > 3:
|
153 |
-
raise ValueError(
|
154 |
-
"Input size must be a three dimensional tensor. Got {}".format(
|
155 |
-
rotation_matrix.shape))
|
156 |
-
if not rotation_matrix.shape[-2:] == (3, 4):
|
157 |
-
raise ValueError(
|
158 |
-
"Input size must be a N x 3 x 4 tensor. Got {}".format(
|
159 |
-
rotation_matrix.shape))
|
160 |
-
|
161 |
-
rmat_t = torch.transpose(rotation_matrix, 1, 2)
|
162 |
-
|
163 |
-
mask_d2 = rmat_t[:, 2, 2] < eps
|
164 |
-
|
165 |
-
mask_d0_d1 = rmat_t[:, 0, 0] > rmat_t[:, 1, 1]
|
166 |
-
mask_d0_nd1 = rmat_t[:, 0, 0] < -rmat_t[:, 1, 1]
|
167 |
-
|
168 |
-
t0 = 1 + rmat_t[:, 0, 0] - rmat_t[:, 1, 1] - rmat_t[:, 2, 2]
|
169 |
-
q0 = torch.stack([
|
170 |
-
rmat_t[:, 1, 2] - rmat_t[:, 2, 1], t0,
|
171 |
-
rmat_t[:, 0, 1] + rmat_t[:, 1, 0], rmat_t[:, 2, 0] + rmat_t[:, 0, 2]
|
172 |
-
], -1)
|
173 |
-
t0_rep = t0.repeat(4, 1).t()
|
174 |
-
|
175 |
-
t1 = 1 - rmat_t[:, 0, 0] + rmat_t[:, 1, 1] - rmat_t[:, 2, 2]
|
176 |
-
q1 = torch.stack([
|
177 |
-
rmat_t[:, 2, 0] - rmat_t[:, 0, 2], rmat_t[:, 0, 1] + rmat_t[:, 1, 0],
|
178 |
-
t1, rmat_t[:, 1, 2] + rmat_t[:, 2, 1]
|
179 |
-
], -1)
|
180 |
-
t1_rep = t1.repeat(4, 1).t()
|
181 |
-
|
182 |
-
t2 = 1 - rmat_t[:, 0, 0] - rmat_t[:, 1, 1] + rmat_t[:, 2, 2]
|
183 |
-
q2 = torch.stack([
|
184 |
-
rmat_t[:, 0, 1] - rmat_t[:, 1, 0], rmat_t[:, 2, 0] + rmat_t[:, 0, 2],
|
185 |
-
rmat_t[:, 1, 2] + rmat_t[:, 2, 1], t2
|
186 |
-
], -1)
|
187 |
-
t2_rep = t2.repeat(4, 1).t()
|
188 |
-
|
189 |
-
t3 = 1 + rmat_t[:, 0, 0] + rmat_t[:, 1, 1] + rmat_t[:, 2, 2]
|
190 |
-
q3 = torch.stack([
|
191 |
-
t3, rmat_t[:, 1, 2] - rmat_t[:, 2, 1],
|
192 |
-
rmat_t[:, 2, 0] - rmat_t[:, 0, 2], rmat_t[:, 0, 1] - rmat_t[:, 1, 0]
|
193 |
-
], -1)
|
194 |
-
t3_rep = t3.repeat(4, 1).t()
|
195 |
-
|
196 |
-
mask_c0 = mask_d2 * mask_d0_d1
|
197 |
-
mask_c1 = mask_d2 * ~mask_d0_d1
|
198 |
-
mask_c2 = ~mask_d2 * mask_d0_nd1
|
199 |
-
mask_c3 = ~mask_d2 * ~mask_d0_nd1
|
200 |
-
mask_c0 = mask_c0.view(-1, 1).type_as(q0)
|
201 |
-
mask_c1 = mask_c1.view(-1, 1).type_as(q1)
|
202 |
-
mask_c2 = mask_c2.view(-1, 1).type_as(q2)
|
203 |
-
mask_c3 = mask_c3.view(-1, 1).type_as(q3)
|
204 |
-
|
205 |
-
q = q0 * mask_c0 + q1 * mask_c1 + q2 * mask_c2 + q3 * mask_c3
|
206 |
-
q /= torch.sqrt(t0_rep * mask_c0 + t1_rep * mask_c1 + # noqa
|
207 |
-
t2_rep * mask_c2 + t3_rep * mask_c3) # noqa
|
208 |
-
q *= 0.5
|
209 |
-
return q
|
210 |
-
|
211 |
-
|
212 |
-
def rot6d_to_rotmat(x):
|
213 |
-
"""Convert 6D rotation representation to 3x3 rotation matrix.
|
214 |
-
Based on Zhou et al., "On the Continuity of Rotation Representations in Neural Networks", CVPR 2019
|
215 |
-
Input:
|
216 |
-
(B,6) Batch of 6-D rotation representations
|
217 |
-
Output:
|
218 |
-
(B,3,3) Batch of corresponding rotation matrices
|
219 |
-
"""
|
220 |
-
x = x.view(-1, 3, 2)
|
221 |
-
a1 = x[:, :, 0]
|
222 |
-
a2 = x[:, :, 1]
|
223 |
-
b1 = F.normalize(a1)
|
224 |
-
b2 = F.normalize(a2 - torch.einsum('bi,bi->b', b1, a2).unsqueeze(-1) * b1)
|
225 |
-
b3 = torch.cross(b1, b2)
|
226 |
-
return torch.stack((b1, b2, b3), dim=-1)
|
227 |
-
|
228 |
-
|
229 |
-
def projection(pred_joints, pred_camera, retain_z=False):
|
230 |
-
pred_cam_t = torch.stack([
|
231 |
-
pred_camera[:, 1], pred_camera[:, 2], 2 * 5000. /
|
232 |
-
(224. * pred_camera[:, 0] + 1e-9)
|
233 |
-
],
|
234 |
-
dim=-1)
|
235 |
-
batch_size = pred_joints.shape[0]
|
236 |
-
camera_center = torch.zeros(batch_size, 2)
|
237 |
-
pred_keypoints_2d = perspective_projection(
|
238 |
-
pred_joints,
|
239 |
-
rotation=torch.eye(3).unsqueeze(0).expand(batch_size, -1,
|
240 |
-
-1).to(pred_joints.device),
|
241 |
-
translation=pred_cam_t,
|
242 |
-
focal_length=5000.,
|
243 |
-
camera_center=camera_center,
|
244 |
-
retain_z=retain_z)
|
245 |
-
# Normalize keypoints to [-1,1]
|
246 |
-
pred_keypoints_2d = pred_keypoints_2d / (224. / 2.)
|
247 |
-
return pred_keypoints_2d
|
248 |
-
|
249 |
-
|
250 |
-
def perspective_projection(points,
|
251 |
-
rotation,
|
252 |
-
translation,
|
253 |
-
focal_length,
|
254 |
-
camera_center,
|
255 |
-
retain_z=False):
|
256 |
-
"""
|
257 |
-
This function computes the perspective projection of a set of points.
|
258 |
-
Input:
|
259 |
-
points (bs, N, 3): 3D points
|
260 |
-
rotation (bs, 3, 3): Camera rotation
|
261 |
-
translation (bs, 3): Camera translation
|
262 |
-
focal_length (bs,) or scalar: Focal length
|
263 |
-
camera_center (bs, 2): Camera center
|
264 |
-
"""
|
265 |
-
batch_size = points.shape[0]
|
266 |
-
K = torch.zeros([batch_size, 3, 3], device=points.device)
|
267 |
-
K[:, 0, 0] = focal_length
|
268 |
-
K[:, 1, 1] = focal_length
|
269 |
-
K[:, 2, 2] = 1.
|
270 |
-
K[:, :-1, -1] = camera_center
|
271 |
-
|
272 |
-
# Transform points
|
273 |
-
points = torch.einsum('bij,bkj->bki', rotation, points)
|
274 |
-
points = points + translation.unsqueeze(1)
|
275 |
-
|
276 |
-
# Apply perspective distortion
|
277 |
-
projected_points = points / points[:, :, -1].unsqueeze(-1)
|
278 |
-
|
279 |
-
# Apply camera intrinsics
|
280 |
-
projected_points = torch.einsum('bij,bkj->bki', K, projected_points)
|
281 |
-
|
282 |
-
if retain_z:
|
283 |
-
return projected_points
|
284 |
-
else:
|
285 |
-
return projected_points[:, :, :-1]
|
286 |
-
|
287 |
-
|
288 |
-
def estimate_translation_np(S,
|
289 |
-
joints_2d,
|
290 |
-
joints_conf,
|
291 |
-
focal_length=5000,
|
292 |
-
img_size=224):
|
293 |
-
"""Find camera translation that brings 3D joints S closest to 2D the corresponding joints_2d.
|
294 |
-
Input:
|
295 |
-
S: (25, 3) 3D joint locations
|
296 |
-
joints: (25, 3) 2D joint locations and confidence
|
297 |
-
Returns:
|
298 |
-
(3,) camera translation vector
|
299 |
-
"""
|
300 |
-
|
301 |
-
num_joints = S.shape[0]
|
302 |
-
# focal length
|
303 |
-
f = np.array([focal_length, focal_length])
|
304 |
-
# optical center
|
305 |
-
center = np.array([img_size / 2., img_size / 2.])
|
306 |
-
|
307 |
-
# transformations
|
308 |
-
Z = np.reshape(np.tile(S[:, 2], (2, 1)).T, -1)
|
309 |
-
XY = np.reshape(S[:, 0:2], -1)
|
310 |
-
O = np.tile(center, num_joints)
|
311 |
-
F = np.tile(f, num_joints)
|
312 |
-
weight2 = np.reshape(np.tile(np.sqrt(joints_conf), (2, 1)).T, -1)
|
313 |
-
|
314 |
-
# least squares
|
315 |
-
Q = np.array([
|
316 |
-
F * np.tile(np.array([1, 0]), num_joints),
|
317 |
-
F * np.tile(np.array([0, 1]), num_joints),
|
318 |
-
O - np.reshape(joints_2d, -1)
|
319 |
-
]).T
|
320 |
-
c = (np.reshape(joints_2d, -1) - O) * Z - F * XY
|
321 |
-
|
322 |
-
# weighted least squares
|
323 |
-
W = np.diagflat(weight2)
|
324 |
-
Q = np.dot(W, Q)
|
325 |
-
c = np.dot(W, c)
|
326 |
-
|
327 |
-
# square matrix
|
328 |
-
A = np.dot(Q.T, Q)
|
329 |
-
b = np.dot(Q.T, c)
|
330 |
-
|
331 |
-
# solution
|
332 |
-
trans = np.linalg.solve(A, b)
|
333 |
-
|
334 |
-
return trans
|
335 |
-
|
336 |
-
|
337 |
-
def estimate_translation(S, joints_2d, focal_length=5000., img_size=224.):
|
338 |
-
"""Find camera translation that brings 3D joints S closest to 2D the corresponding joints_2d.
|
339 |
-
Input:
|
340 |
-
S: (B, 49, 3) 3D joint locations
|
341 |
-
joints: (B, 49, 3) 2D joint locations and confidence
|
342 |
-
Returns:
|
343 |
-
(B, 3) camera translation vectors
|
344 |
-
"""
|
345 |
-
|
346 |
-
device = S.device
|
347 |
-
# Use only joints 25:49 (GT joints)
|
348 |
-
S = S[:, 25:, :].cpu().numpy()
|
349 |
-
joints_2d = joints_2d[:, 25:, :].cpu().numpy()
|
350 |
-
joints_conf = joints_2d[:, :, -1]
|
351 |
-
joints_2d = joints_2d[:, :, :-1]
|
352 |
-
trans = np.zeros((S.shape[0], 3), dtype=np.float32)
|
353 |
-
# Find the translation for each example in the batch
|
354 |
-
for i in range(S.shape[0]):
|
355 |
-
S_i = S[i]
|
356 |
-
joints_i = joints_2d[i]
|
357 |
-
conf_i = joints_conf[i]
|
358 |
-
trans[i] = estimate_translation_np(S_i,
|
359 |
-
joints_i,
|
360 |
-
conf_i,
|
361 |
-
focal_length=focal_length,
|
362 |
-
img_size=img_size)
|
363 |
-
return torch.from_numpy(trans).to(device)
|
364 |
-
|
365 |
-
|
366 |
-
def Rot_y(angle, category='torch', prepend_dim=True, device=None):
|
367 |
-
'''Rotate around y-axis by angle
|
368 |
-
Args:
|
369 |
-
category: 'torch' or 'numpy'
|
370 |
-
prepend_dim: prepend an extra dimension
|
371 |
-
Return: Rotation matrix with shape [1, 3, 3] (prepend_dim=True)
|
372 |
-
'''
|
373 |
-
m = np.array([[np.cos(angle), 0., np.sin(angle)], [0., 1., 0.],
|
374 |
-
[-np.sin(angle), 0., np.cos(angle)]])
|
375 |
-
if category == 'torch':
|
376 |
-
if prepend_dim:
|
377 |
-
return torch.tensor(m, dtype=torch.float,
|
378 |
-
device=device).unsqueeze(0)
|
379 |
-
else:
|
380 |
-
return torch.tensor(m, dtype=torch.float, device=device)
|
381 |
-
elif category == 'numpy':
|
382 |
-
if prepend_dim:
|
383 |
-
return np.expand_dims(m, 0)
|
384 |
-
else:
|
385 |
-
return m
|
386 |
-
else:
|
387 |
-
raise ValueError("category must be 'torch' or 'numpy'")
|
388 |
-
|
389 |
-
|
390 |
-
def Rot_x(angle, category='torch', prepend_dim=True, device=None):
|
391 |
-
'''Rotate around x-axis by angle
|
392 |
-
Args:
|
393 |
-
category: 'torch' or 'numpy'
|
394 |
-
prepend_dim: prepend an extra dimension
|
395 |
-
Return: Rotation matrix with shape [1, 3, 3] (prepend_dim=True)
|
396 |
-
'''
|
397 |
-
m = np.array([[1., 0., 0.], [0., np.cos(angle), -np.sin(angle)],
|
398 |
-
[0., np.sin(angle), np.cos(angle)]])
|
399 |
-
if category == 'torch':
|
400 |
-
if prepend_dim:
|
401 |
-
return torch.tensor(m, dtype=torch.float,
|
402 |
-
device=device).unsqueeze(0)
|
403 |
-
else:
|
404 |
-
return torch.tensor(m, dtype=torch.float, device=device)
|
405 |
-
elif category == 'numpy':
|
406 |
-
if prepend_dim:
|
407 |
-
return np.expand_dims(m, 0)
|
408 |
-
else:
|
409 |
-
return m
|
410 |
-
else:
|
411 |
-
raise ValueError("category must be 'torch' or 'numpy'")
|
412 |
-
|
413 |
-
|
414 |
-
def Rot_z(angle, category='torch', prepend_dim=True, device=None):
|
415 |
-
'''Rotate around z-axis by angle
|
416 |
-
Args:
|
417 |
-
category: 'torch' or 'numpy'
|
418 |
-
prepend_dim: prepend an extra dimension
|
419 |
-
Return: Rotation matrix with shape [1, 3, 3] (prepend_dim=True)
|
420 |
-
'''
|
421 |
-
m = np.array([[np.cos(angle), -np.sin(angle), 0.],
|
422 |
-
[np.sin(angle), np.cos(angle), 0.], [0., 0., 1.]])
|
423 |
-
if category == 'torch':
|
424 |
-
if prepend_dim:
|
425 |
-
return torch.tensor(m, dtype=torch.float,
|
426 |
-
device=device).unsqueeze(0)
|
427 |
-
else:
|
428 |
-
return torch.tensor(m, dtype=torch.float, device=device)
|
429 |
-
elif category == 'numpy':
|
430 |
-
if prepend_dim:
|
431 |
-
return np.expand_dims(m, 0)
|
432 |
-
else:
|
433 |
-
return m
|
434 |
-
else:
|
435 |
-
raise ValueError("category must be 'torch' or 'numpy'")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /utils / imutils.py
DELETED
@@ -1,491 +0,0 @@
|
|
1 |
-
"""
|
2 |
-
This file contains functions that are used to perform data augmentation.
|
3 |
-
"""
|
4 |
-
import cv2
|
5 |
-
import io
|
6 |
-
import torch
|
7 |
-
import numpy as np
|
8 |
-
from PIL import Image
|
9 |
-
from rembg import remove
|
10 |
-
from rembg.session_factory import new_session
|
11 |
-
from torchvision.models import detection
|
12 |
-
|
13 |
-
from lib.pymaf.core import constants
|
14 |
-
from lib.pymaf.utils.streamer import aug_matrix
|
15 |
-
from lib.common.cloth_extraction import load_segmentation
|
16 |
-
from torchvision import transforms
|
17 |
-
|
18 |
-
|
19 |
-
def load_img(img_file):
|
20 |
-
|
21 |
-
img = cv2.imread(img_file, cv2.IMREAD_UNCHANGED)
|
22 |
-
if len(img.shape) == 2:
|
23 |
-
img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
|
24 |
-
|
25 |
-
if not img_file.endswith("png"):
|
26 |
-
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
27 |
-
else:
|
28 |
-
img = cv2.cvtColor(img, cv2.COLOR_RGBA2BGR)
|
29 |
-
|
30 |
-
return img
|
31 |
-
|
32 |
-
|
33 |
-
def get_bbox(img, det):
|
34 |
-
|
35 |
-
input = np.float32(img)
|
36 |
-
input = (input / 255.0 -
|
37 |
-
(0.5, 0.5, 0.5)) / (0.5, 0.5, 0.5) # TO [-1.0, 1.0]
|
38 |
-
input = input.transpose(2, 0, 1) # TO [3 x H x W]
|
39 |
-
bboxes, probs = det(torch.from_numpy(input).float().unsqueeze(0))
|
40 |
-
|
41 |
-
probs = probs.unsqueeze(3)
|
42 |
-
bboxes = (bboxes * probs).sum(dim=1, keepdim=True) / probs.sum(
|
43 |
-
dim=1, keepdim=True)
|
44 |
-
bbox = bboxes[0, 0, 0].cpu().numpy()
|
45 |
-
|
46 |
-
return bbox
|
47 |
-
# Michael Black is
|
48 |
-
|
49 |
-
|
50 |
-
def get_transformer(input_res):
|
51 |
-
|
52 |
-
image_to_tensor = transforms.Compose([
|
53 |
-
transforms.Resize(input_res),
|
54 |
-
transforms.ToTensor(),
|
55 |
-
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
|
56 |
-
])
|
57 |
-
|
58 |
-
mask_to_tensor = transforms.Compose([
|
59 |
-
transforms.Resize(input_res),
|
60 |
-
transforms.ToTensor(),
|
61 |
-
transforms.Normalize((0.0, ), (1.0, ))
|
62 |
-
])
|
63 |
-
|
64 |
-
image_to_pymaf_tensor = transforms.Compose([
|
65 |
-
transforms.Resize(size=224),
|
66 |
-
transforms.Normalize(mean=constants.IMG_NORM_MEAN,
|
67 |
-
std=constants.IMG_NORM_STD)
|
68 |
-
])
|
69 |
-
|
70 |
-
image_to_pixie_tensor = transforms.Compose([
|
71 |
-
transforms.Resize(224)
|
72 |
-
])
|
73 |
-
|
74 |
-
def image_to_hybrik_tensor(img):
|
75 |
-
# mean
|
76 |
-
img[0].add_(-0.406)
|
77 |
-
img[1].add_(-0.457)
|
78 |
-
img[2].add_(-0.480)
|
79 |
-
|
80 |
-
# std
|
81 |
-
img[0].div_(0.225)
|
82 |
-
img[1].div_(0.224)
|
83 |
-
img[2].div_(0.229)
|
84 |
-
return img
|
85 |
-
|
86 |
-
return [image_to_tensor, mask_to_tensor, image_to_pymaf_tensor, image_to_pixie_tensor, image_to_hybrik_tensor]
|
87 |
-
|
88 |
-
|
89 |
-
def process_image(img_file, hps_type, input_res=512, device=None, seg_path=None):
|
90 |
-
"""Read image, do preprocessing and possibly crop it according to the bounding box.
|
91 |
-
If there are bounding box annotations, use them to crop the image.
|
92 |
-
If no bounding box is specified but openpose detections are available, use them to get the bounding box.
|
93 |
-
"""
|
94 |
-
|
95 |
-
[image_to_tensor, mask_to_tensor, image_to_pymaf_tensor,
|
96 |
-
image_to_pixie_tensor, image_to_hybrik_tensor] = get_transformer(input_res)
|
97 |
-
|
98 |
-
img_ori = load_img(img_file)
|
99 |
-
|
100 |
-
in_height, in_width, _ = img_ori.shape
|
101 |
-
M = aug_matrix(in_width, in_height, input_res*2, input_res*2)
|
102 |
-
|
103 |
-
# from rectangle to square
|
104 |
-
img_for_crop = cv2.warpAffine(img_ori, M[0:2, :],
|
105 |
-
(input_res*2, input_res*2), flags=cv2.INTER_CUBIC)
|
106 |
-
|
107 |
-
# detection for bbox
|
108 |
-
detector = detection.maskrcnn_resnet50_fpn(pretrained=True)
|
109 |
-
detector.eval()
|
110 |
-
predictions = detector(
|
111 |
-
[torch.from_numpy(img_for_crop).permute(2, 0, 1) / 255.])[0]
|
112 |
-
human_ids = torch.where(
|
113 |
-
predictions["scores"] == predictions["scores"][predictions['labels'] == 1].max())
|
114 |
-
bbox = predictions["boxes"][human_ids, :].flatten().detach().cpu().numpy()
|
115 |
-
|
116 |
-
width = bbox[2] - bbox[0]
|
117 |
-
height = bbox[3] - bbox[1]
|
118 |
-
center = np.array([(bbox[0] + bbox[2]) / 2.0,
|
119 |
-
(bbox[1] + bbox[3]) / 2.0])
|
120 |
-
|
121 |
-
scale = max(height, width) / 180
|
122 |
-
|
123 |
-
if hps_type == 'hybrik':
|
124 |
-
img_np = crop_for_hybrik(img_for_crop, center,
|
125 |
-
np.array([scale * 180, scale * 180]))
|
126 |
-
else:
|
127 |
-
img_np, cropping_parameters = crop(
|
128 |
-
img_for_crop, center, scale, (input_res, input_res))
|
129 |
-
|
130 |
-
img_pil = Image.fromarray(remove(img_np, post_process_mask=True, session=new_session("u2net")))
|
131 |
-
|
132 |
-
# for icon
|
133 |
-
img_rgb = image_to_tensor(img_pil.convert("RGB"))
|
134 |
-
img_mask = torch.tensor(1.0) - (mask_to_tensor(img_pil.split()[-1]) <
|
135 |
-
torch.tensor(0.5)).float()
|
136 |
-
img_tensor = img_rgb * img_mask
|
137 |
-
|
138 |
-
# for hps
|
139 |
-
img_hps = img_np.astype(np.float32) / 255.
|
140 |
-
img_hps = torch.from_numpy(img_hps).permute(2, 0, 1)
|
141 |
-
|
142 |
-
if hps_type == 'bev':
|
143 |
-
img_hps = img_np[:, :, [2, 1, 0]]
|
144 |
-
elif hps_type == 'hybrik':
|
145 |
-
img_hps = image_to_hybrik_tensor(img_hps).unsqueeze(0).to(device)
|
146 |
-
elif hps_type != 'pixie':
|
147 |
-
img_hps = image_to_pymaf_tensor(img_hps).unsqueeze(0).to(device)
|
148 |
-
else:
|
149 |
-
img_hps = image_to_pixie_tensor(img_hps).unsqueeze(0).to(device)
|
150 |
-
|
151 |
-
# uncrop params
|
152 |
-
uncrop_param = {'center': center,
|
153 |
-
'scale': scale,
|
154 |
-
'ori_shape': img_ori.shape,
|
155 |
-
'box_shape': img_np.shape,
|
156 |
-
'crop_shape': img_for_crop.shape,
|
157 |
-
'M': M}
|
158 |
-
|
159 |
-
if not (seg_path is None):
|
160 |
-
segmentations = load_segmentation(seg_path, (in_height, in_width))
|
161 |
-
seg_coord_normalized = []
|
162 |
-
for seg in segmentations:
|
163 |
-
coord_normalized = []
|
164 |
-
for xy in seg['coordinates']:
|
165 |
-
xy_h = np.vstack((xy[:, 0], xy[:, 1], np.ones(len(xy)))).T
|
166 |
-
warped_indeces = M[0:2, :] @ xy_h[:, :, None]
|
167 |
-
warped_indeces = np.array(warped_indeces).astype(int)
|
168 |
-
warped_indeces.resize((warped_indeces.shape[:2]))
|
169 |
-
|
170 |
-
# cropped_indeces = crop_segmentation(warped_indeces, center, scale, (input_res, input_res), img_np.shape)
|
171 |
-
cropped_indeces = crop_segmentation(
|
172 |
-
warped_indeces, (input_res, input_res), cropping_parameters)
|
173 |
-
|
174 |
-
indices = np.vstack(
|
175 |
-
(cropped_indeces[:, 0], cropped_indeces[:, 1])).T
|
176 |
-
|
177 |
-
# Convert to NDC coordinates
|
178 |
-
seg_cropped_normalized = 2*(indices / input_res) - 1
|
179 |
-
# Don't know why we need to divide by 50 but it works ¯\_(ツ)_/¯ (probably some scaling factor somewhere)
|
180 |
-
# Divide only by 45 on the horizontal axis to take the curve of the human body into account
|
181 |
-
seg_cropped_normalized[:, 0] = (
|
182 |
-
1/40) * seg_cropped_normalized[:, 0]
|
183 |
-
seg_cropped_normalized[:, 1] = (
|
184 |
-
1/50) * seg_cropped_normalized[:, 1]
|
185 |
-
coord_normalized.append(seg_cropped_normalized)
|
186 |
-
|
187 |
-
seg['coord_normalized'] = coord_normalized
|
188 |
-
seg_coord_normalized.append(seg)
|
189 |
-
|
190 |
-
return img_tensor, img_hps, img_ori, img_mask, uncrop_param, seg_coord_normalized
|
191 |
-
|
192 |
-
return img_tensor, img_hps, img_ori, img_mask, uncrop_param
|
193 |
-
|
194 |
-
|
195 |
-
def get_transform(center, scale, res):
|
196 |
-
"""Generate transformation matrix."""
|
197 |
-
h = 200 * scale
|
198 |
-
t = np.zeros((3, 3))
|
199 |
-
t[0, 0] = float(res[1]) / h
|
200 |
-
t[1, 1] = float(res[0]) / h
|
201 |
-
t[0, 2] = res[1] * (-float(center[0]) / h + .5)
|
202 |
-
t[1, 2] = res[0] * (-float(center[1]) / h + .5)
|
203 |
-
t[2, 2] = 1
|
204 |
-
|
205 |
-
return t
|
206 |
-
|
207 |
-
|
208 |
-
def transform(pt, center, scale, res, invert=0):
|
209 |
-
"""Transform pixel location to different reference."""
|
210 |
-
t = get_transform(center, scale, res)
|
211 |
-
if invert:
|
212 |
-
t = np.linalg.inv(t)
|
213 |
-
new_pt = np.array([pt[0] - 1, pt[1] - 1, 1.]).T
|
214 |
-
new_pt = np.dot(t, new_pt)
|
215 |
-
return np.around(new_pt[:2]).astype(np.int16)
|
216 |
-
|
217 |
-
|
218 |
-
def crop(img, center, scale, res):
|
219 |
-
"""Crop image according to the supplied bounding box."""
|
220 |
-
|
221 |
-
# Upper left point
|
222 |
-
ul = np.array(transform([0, 0], center, scale, res, invert=1))
|
223 |
-
|
224 |
-
# Bottom right point
|
225 |
-
br = np.array(transform(res, center, scale, res, invert=1))
|
226 |
-
|
227 |
-
new_shape = [br[1] - ul[1], br[0] - ul[0]]
|
228 |
-
if len(img.shape) > 2:
|
229 |
-
new_shape += [img.shape[2]]
|
230 |
-
new_img = np.zeros(new_shape)
|
231 |
-
|
232 |
-
# Range to fill new array
|
233 |
-
new_x = max(0, -ul[0]), min(br[0], len(img[0])) - ul[0]
|
234 |
-
new_y = max(0, -ul[1]), min(br[1], len(img)) - ul[1]
|
235 |
-
|
236 |
-
# Range to sample from original image
|
237 |
-
old_x = max(0, ul[0]), min(len(img[0]), br[0])
|
238 |
-
old_y = max(0, ul[1]), min(len(img), br[1])
|
239 |
-
|
240 |
-
new_img[new_y[0]:new_y[1], new_x[0]:new_x[1]
|
241 |
-
] = img[old_y[0]:old_y[1], old_x[0]:old_x[1]]
|
242 |
-
if len(img.shape) == 2:
|
243 |
-
new_img = np.array(Image.fromarray(new_img).resize(res))
|
244 |
-
else:
|
245 |
-
new_img = np.array(Image.fromarray(
|
246 |
-
new_img.astype(np.uint8)).resize(res))
|
247 |
-
|
248 |
-
return new_img, (old_x, new_x, old_y, new_y, new_shape)
|
249 |
-
|
250 |
-
|
251 |
-
def crop_segmentation(org_coord, res, cropping_parameters):
|
252 |
-
old_x, new_x, old_y, new_y, new_shape = cropping_parameters
|
253 |
-
|
254 |
-
new_coord = np.zeros((org_coord.shape))
|
255 |
-
new_coord[:, 0] = new_x[0] + (org_coord[:, 0] - old_x[0])
|
256 |
-
new_coord[:, 1] = new_y[0] + (org_coord[:, 1] - old_y[0])
|
257 |
-
|
258 |
-
new_coord[:, 0] = res[0] * (new_coord[:, 0] / new_shape[1])
|
259 |
-
new_coord[:, 1] = res[1] * (new_coord[:, 1] / new_shape[0])
|
260 |
-
|
261 |
-
return new_coord
|
262 |
-
|
263 |
-
|
264 |
-
def crop_for_hybrik(img, center, scale):
|
265 |
-
inp_h, inp_w = (256, 256)
|
266 |
-
trans = get_affine_transform(center, scale, 0, [inp_w, inp_h])
|
267 |
-
new_img = cv2.warpAffine(
|
268 |
-
img, trans, (int(inp_w), int(inp_h)), flags=cv2.INTER_LINEAR)
|
269 |
-
return new_img
|
270 |
-
|
271 |
-
|
272 |
-
def get_affine_transform(center,
|
273 |
-
scale,
|
274 |
-
rot,
|
275 |
-
output_size,
|
276 |
-
shift=np.array([0, 0], dtype=np.float32),
|
277 |
-
inv=0):
|
278 |
-
|
279 |
-
def get_dir(src_point, rot_rad):
|
280 |
-
"""Rotate the point by `rot_rad` degree."""
|
281 |
-
sn, cs = np.sin(rot_rad), np.cos(rot_rad)
|
282 |
-
|
283 |
-
src_result = [0, 0]
|
284 |
-
src_result[0] = src_point[0] * cs - src_point[1] * sn
|
285 |
-
src_result[1] = src_point[0] * sn + src_point[1] * cs
|
286 |
-
|
287 |
-
return src_result
|
288 |
-
|
289 |
-
def get_3rd_point(a, b):
|
290 |
-
"""Return vector c that perpendicular to (a - b)."""
|
291 |
-
direct = a - b
|
292 |
-
return b + np.array([-direct[1], direct[0]], dtype=np.float32)
|
293 |
-
|
294 |
-
if not isinstance(scale, np.ndarray) and not isinstance(scale, list):
|
295 |
-
scale = np.array([scale, scale])
|
296 |
-
|
297 |
-
scale_tmp = scale
|
298 |
-
src_w = scale_tmp[0]
|
299 |
-
dst_w = output_size[0]
|
300 |
-
dst_h = output_size[1]
|
301 |
-
|
302 |
-
rot_rad = np.pi * rot / 180
|
303 |
-
src_dir = get_dir([0, src_w * -0.5], rot_rad)
|
304 |
-
dst_dir = np.array([0, dst_w * -0.5], np.float32)
|
305 |
-
|
306 |
-
src = np.zeros((3, 2), dtype=np.float32)
|
307 |
-
dst = np.zeros((3, 2), dtype=np.float32)
|
308 |
-
src[0, :] = center + scale_tmp * shift
|
309 |
-
src[1, :] = center + src_dir + scale_tmp * shift
|
310 |
-
dst[0, :] = [dst_w * 0.5, dst_h * 0.5]
|
311 |
-
dst[1, :] = np.array([dst_w * 0.5, dst_h * 0.5]) + dst_dir
|
312 |
-
|
313 |
-
src[2:, :] = get_3rd_point(src[0, :], src[1, :])
|
314 |
-
dst[2:, :] = get_3rd_point(dst[0, :], dst[1, :])
|
315 |
-
|
316 |
-
if inv:
|
317 |
-
trans = cv2.getAffineTransform(np.float32(dst), np.float32(src))
|
318 |
-
else:
|
319 |
-
trans = cv2.getAffineTransform(np.float32(src), np.float32(dst))
|
320 |
-
|
321 |
-
return trans
|
322 |
-
|
323 |
-
|
324 |
-
def corner_align(ul, br):
|
325 |
-
|
326 |
-
if ul[1]-ul[0] != br[1]-br[0]:
|
327 |
-
ul[1] = ul[0]+br[1]-br[0]
|
328 |
-
|
329 |
-
return ul, br
|
330 |
-
|
331 |
-
|
332 |
-
def uncrop(img, center, scale, orig_shape):
|
333 |
-
"""'Undo' the image cropping/resizing.
|
334 |
-
This function is used when evaluating mask/part segmentation.
|
335 |
-
"""
|
336 |
-
|
337 |
-
res = img.shape[:2]
|
338 |
-
|
339 |
-
# Upper left point
|
340 |
-
ul = np.array(transform([0, 0], center, scale, res, invert=1))
|
341 |
-
# Bottom right point
|
342 |
-
br = np.array(transform(res, center, scale, res, invert=1))
|
343 |
-
|
344 |
-
# quick fix
|
345 |
-
ul, br = corner_align(ul, br)
|
346 |
-
|
347 |
-
# size of cropped image
|
348 |
-
crop_shape = [br[1] - ul[1], br[0] - ul[0]]
|
349 |
-
new_img = np.zeros(orig_shape, dtype=np.uint8)
|
350 |
-
|
351 |
-
# Range to fill new array
|
352 |
-
new_x = max(0, -ul[0]), min(br[0], orig_shape[1]) - ul[0]
|
353 |
-
new_y = max(0, -ul[1]), min(br[1], orig_shape[0]) - ul[1]
|
354 |
-
|
355 |
-
# Range to sample from original image
|
356 |
-
old_x = max(0, ul[0]), min(orig_shape[1], br[0])
|
357 |
-
old_y = max(0, ul[1]), min(orig_shape[0], br[1])
|
358 |
-
|
359 |
-
img = np.array(Image.fromarray(img.astype(np.uint8)).resize(crop_shape))
|
360 |
-
|
361 |
-
new_img[old_y[0]:old_y[1], old_x[0]:old_x[1]
|
362 |
-
] = img[new_y[0]:new_y[1], new_x[0]:new_x[1]]
|
363 |
-
|
364 |
-
return new_img
|
365 |
-
|
366 |
-
|
367 |
-
def rot_aa(aa, rot):
|
368 |
-
"""Rotate axis angle parameters."""
|
369 |
-
# pose parameters
|
370 |
-
R = np.array([[np.cos(np.deg2rad(-rot)), -np.sin(np.deg2rad(-rot)), 0],
|
371 |
-
[np.sin(np.deg2rad(-rot)),
|
372 |
-
np.cos(np.deg2rad(-rot)), 0], [0, 0, 1]])
|
373 |
-
# find the rotation of the body in camera frame
|
374 |
-
per_rdg, _ = cv2.Rodrigues(aa)
|
375 |
-
# apply the global rotation to the global orientation
|
376 |
-
resrot, _ = cv2.Rodrigues(np.dot(R, per_rdg))
|
377 |
-
aa = (resrot.T)[0]
|
378 |
-
return aa
|
379 |
-
|
380 |
-
|
381 |
-
def flip_img(img):
|
382 |
-
"""Flip rgb images or masks.
|
383 |
-
channels come last, e.g. (256,256,3).
|
384 |
-
"""
|
385 |
-
img = np.fliplr(img)
|
386 |
-
return img
|
387 |
-
|
388 |
-
|
389 |
-
def flip_kp(kp, is_smpl=False):
|
390 |
-
"""Flip keypoints."""
|
391 |
-
if len(kp) == 24:
|
392 |
-
if is_smpl:
|
393 |
-
flipped_parts = constants.SMPL_JOINTS_FLIP_PERM
|
394 |
-
else:
|
395 |
-
flipped_parts = constants.J24_FLIP_PERM
|
396 |
-
elif len(kp) == 49:
|
397 |
-
if is_smpl:
|
398 |
-
flipped_parts = constants.SMPL_J49_FLIP_PERM
|
399 |
-
else:
|
400 |
-
flipped_parts = constants.J49_FLIP_PERM
|
401 |
-
kp = kp[flipped_parts]
|
402 |
-
kp[:, 0] = -kp[:, 0]
|
403 |
-
return kp
|
404 |
-
|
405 |
-
|
406 |
-
def flip_pose(pose):
|
407 |
-
"""Flip pose.
|
408 |
-
The flipping is based on SMPL parameters.
|
409 |
-
"""
|
410 |
-
flipped_parts = constants.SMPL_POSE_FLIP_PERM
|
411 |
-
pose = pose[flipped_parts]
|
412 |
-
# we also negate the second and the third dimension of the axis-angle
|
413 |
-
pose[1::3] = -pose[1::3]
|
414 |
-
pose[2::3] = -pose[2::3]
|
415 |
-
return pose
|
416 |
-
|
417 |
-
|
418 |
-
def normalize_2d_kp(kp_2d, crop_size=224, inv=False):
|
419 |
-
# Normalize keypoints between -1, 1
|
420 |
-
if not inv:
|
421 |
-
ratio = 1.0 / crop_size
|
422 |
-
kp_2d = 2.0 * kp_2d * ratio - 1.0
|
423 |
-
else:
|
424 |
-
ratio = 1.0 / crop_size
|
425 |
-
kp_2d = (kp_2d + 1.0) / (2 * ratio)
|
426 |
-
|
427 |
-
return kp_2d
|
428 |
-
|
429 |
-
|
430 |
-
def generate_heatmap(joints, heatmap_size, sigma=1, joints_vis=None):
|
431 |
-
'''
|
432 |
-
param joints: [num_joints, 3]
|
433 |
-
param joints_vis: [num_joints, 3]
|
434 |
-
return: target, target_weight(1: visible, 0: invisible)
|
435 |
-
'''
|
436 |
-
num_joints = joints.shape[0]
|
437 |
-
device = joints.device
|
438 |
-
cur_device = torch.device(device.type, device.index)
|
439 |
-
if not hasattr(heatmap_size, '__len__'):
|
440 |
-
# width height
|
441 |
-
heatmap_size = [heatmap_size, heatmap_size]
|
442 |
-
assert len(heatmap_size) == 2
|
443 |
-
target_weight = np.ones((num_joints, 1), dtype=np.float32)
|
444 |
-
if joints_vis is not None:
|
445 |
-
target_weight[:, 0] = joints_vis[:, 0]
|
446 |
-
target = torch.zeros((num_joints, heatmap_size[1], heatmap_size[0]),
|
447 |
-
dtype=torch.float32,
|
448 |
-
device=cur_device)
|
449 |
-
|
450 |
-
tmp_size = sigma * 3
|
451 |
-
|
452 |
-
for joint_id in range(num_joints):
|
453 |
-
mu_x = int(joints[joint_id][0] * heatmap_size[0] + 0.5)
|
454 |
-
mu_y = int(joints[joint_id][1] * heatmap_size[1] + 0.5)
|
455 |
-
# Check that any part of the gaussian is in-bounds
|
456 |
-
ul = [int(mu_x - tmp_size), int(mu_y - tmp_size)]
|
457 |
-
br = [int(mu_x + tmp_size + 1), int(mu_y + tmp_size + 1)]
|
458 |
-
if ul[0] >= heatmap_size[0] or ul[1] >= heatmap_size[1] \
|
459 |
-
or br[0] < 0 or br[1] < 0:
|
460 |
-
# If not, just return the image as is
|
461 |
-
target_weight[joint_id] = 0
|
462 |
-
continue
|
463 |
-
|
464 |
-
# # Generate gaussian
|
465 |
-
size = 2 * tmp_size + 1
|
466 |
-
# x = np.arange(0, size, 1, np.float32)
|
467 |
-
# y = x[:, np.newaxis]
|
468 |
-
# x0 = y0 = size // 2
|
469 |
-
# # The gaussian is not normalized, we want the center value to equal 1
|
470 |
-
# g = np.exp(- ((x - x0) ** 2 + (y - y0) ** 2) / (2 * sigma ** 2))
|
471 |
-
# g = torch.from_numpy(g.astype(np.float32))
|
472 |
-
|
473 |
-
x = torch.arange(0, size, dtype=torch.float32, device=cur_device)
|
474 |
-
y = x.unsqueeze(-1)
|
475 |
-
x0 = y0 = size // 2
|
476 |
-
# The gaussian is not normalized, we want the center value to equal 1
|
477 |
-
g = torch.exp(-((x - x0)**2 + (y - y0)**2) / (2 * sigma**2))
|
478 |
-
|
479 |
-
# Usable gaussian range
|
480 |
-
g_x = max(0, -ul[0]), min(br[0], heatmap_size[0]) - ul[0]
|
481 |
-
g_y = max(0, -ul[1]), min(br[1], heatmap_size[1]) - ul[1]
|
482 |
-
# Image range
|
483 |
-
img_x = max(0, ul[0]), min(br[0], heatmap_size[0])
|
484 |
-
img_y = max(0, ul[1]), min(br[1], heatmap_size[1])
|
485 |
-
|
486 |
-
v = target_weight[joint_id]
|
487 |
-
if v > 0.5:
|
488 |
-
target[joint_id][img_y[0]:img_y[1], img_x[0]:img_x[1]] = \
|
489 |
-
g[g_y[0]:g_y[1], g_x[0]:g_x[1]]
|
490 |
-
|
491 |
-
return target, target_weight
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /utils / streamer.py
DELETED
@@ -1,142 +0,0 @@
|
|
1 |
-
import cv2
|
2 |
-
import torch
|
3 |
-
import numpy as np
|
4 |
-
import imageio
|
5 |
-
|
6 |
-
|
7 |
-
def aug_matrix(w1, h1, w2, h2):
|
8 |
-
dx = (w2 - w1) / 2.0
|
9 |
-
dy = (h2 - h1) / 2.0
|
10 |
-
|
11 |
-
matrix_trans = np.array([[1.0, 0, dx],
|
12 |
-
[0, 1.0, dy],
|
13 |
-
[0, 0, 1.0]])
|
14 |
-
|
15 |
-
scale = np.min([float(w2)/w1, float(h2)/h1])
|
16 |
-
|
17 |
-
M = get_affine_matrix(
|
18 |
-
center=(w2 / 2.0, h2 / 2.0),
|
19 |
-
translate=(0, 0),
|
20 |
-
scale=scale)
|
21 |
-
|
22 |
-
M = np.array(M + [0., 0., 1.]).reshape(3, 3)
|
23 |
-
M = M.dot(matrix_trans)
|
24 |
-
|
25 |
-
return M
|
26 |
-
|
27 |
-
|
28 |
-
def get_affine_matrix(center, translate, scale):
|
29 |
-
cx, cy = center
|
30 |
-
tx, ty = translate
|
31 |
-
|
32 |
-
M = [1, 0, 0,
|
33 |
-
0, 1, 0]
|
34 |
-
M = [x * scale for x in M]
|
35 |
-
|
36 |
-
# Apply translation and of center translation: RSS * C^-1
|
37 |
-
M[2] += M[0] * (-cx) + M[1] * (-cy)
|
38 |
-
M[5] += M[3] * (-cx) + M[4] * (-cy)
|
39 |
-
|
40 |
-
# Apply center translation: T * C * RSS * C^-1
|
41 |
-
M[2] += cx + tx
|
42 |
-
M[5] += cy + ty
|
43 |
-
return M
|
44 |
-
|
45 |
-
|
46 |
-
class BaseStreamer():
|
47 |
-
"""This streamer will return images at 512x512 size.
|
48 |
-
"""
|
49 |
-
|
50 |
-
def __init__(self,
|
51 |
-
width=512, height=512, pad=True,
|
52 |
-
mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5),
|
53 |
-
**kwargs):
|
54 |
-
self.width = width
|
55 |
-
self.height = height
|
56 |
-
self.pad = pad
|
57 |
-
self.mean = np.array(mean)
|
58 |
-
self.std = np.array(std)
|
59 |
-
|
60 |
-
self.loader = self.create_loader()
|
61 |
-
|
62 |
-
def create_loader(self):
|
63 |
-
raise NotImplementedError
|
64 |
-
yield np.zeros((600, 400, 3)) # in RGB (0, 255)
|
65 |
-
|
66 |
-
def __getitem__(self, index):
|
67 |
-
image = next(self.loader)
|
68 |
-
in_height, in_width, _ = image.shape
|
69 |
-
M = aug_matrix(in_width, in_height, self.width, self.height, self.pad)
|
70 |
-
image = cv2.warpAffine(
|
71 |
-
image, M[0:2, :], (self.width, self.height), flags=cv2.INTER_CUBIC)
|
72 |
-
|
73 |
-
input = np.float32(image)
|
74 |
-
input = (input / 255.0 - self.mean) / self.std # TO [-1.0, 1.0]
|
75 |
-
input = input.transpose(2, 0, 1) # TO [3 x H x W]
|
76 |
-
return torch.from_numpy(input).float()
|
77 |
-
|
78 |
-
def __len__(self):
|
79 |
-
raise NotImplementedError
|
80 |
-
|
81 |
-
|
82 |
-
class CaptureStreamer(BaseStreamer):
|
83 |
-
"""This streamer takes webcam as input.
|
84 |
-
"""
|
85 |
-
|
86 |
-
def __init__(self, id=0, width=512, height=512, pad=True, **kwargs):
|
87 |
-
super().__init__(width, height, pad, **kwargs)
|
88 |
-
self.capture = cv2.VideoCapture(id)
|
89 |
-
|
90 |
-
def create_loader(self):
|
91 |
-
while True:
|
92 |
-
_, image = self.capture.read()
|
93 |
-
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # RGB
|
94 |
-
yield image
|
95 |
-
|
96 |
-
def __len__(self):
|
97 |
-
return 100_000_000
|
98 |
-
|
99 |
-
def __del__(self):
|
100 |
-
self.capture.release()
|
101 |
-
|
102 |
-
|
103 |
-
class VideoListStreamer(BaseStreamer):
|
104 |
-
"""This streamer takes a list of video files as input.
|
105 |
-
"""
|
106 |
-
|
107 |
-
def __init__(self, files, width=512, height=512, pad=True, **kwargs):
|
108 |
-
super().__init__(width, height, pad, **kwargs)
|
109 |
-
self.files = files
|
110 |
-
self.captures = [imageio.get_reader(f) for f in files]
|
111 |
-
self.nframes = sum([int(cap._meta["fps"] * cap._meta["duration"])
|
112 |
-
for cap in self.captures])
|
113 |
-
|
114 |
-
def create_loader(self):
|
115 |
-
for capture in self.captures:
|
116 |
-
for image in capture: # RGB
|
117 |
-
yield image
|
118 |
-
|
119 |
-
def __len__(self):
|
120 |
-
return self.nframes
|
121 |
-
|
122 |
-
def __del__(self):
|
123 |
-
for capture in self.captures:
|
124 |
-
capture.close()
|
125 |
-
|
126 |
-
|
127 |
-
class ImageListStreamer(BaseStreamer):
|
128 |
-
"""This streamer takes a list of image files as input.
|
129 |
-
"""
|
130 |
-
|
131 |
-
def __init__(self, files, width=512, height=512, pad=True, **kwargs):
|
132 |
-
super().__init__(width, height, pad, **kwargs)
|
133 |
-
self.files = files
|
134 |
-
|
135 |
-
def create_loader(self):
|
136 |
-
for f in self.files:
|
137 |
-
image = cv2.imread(f, cv2.IMREAD_UNCHANGED)[:, :, 0:3]
|
138 |
-
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # RGB
|
139 |
-
yield image
|
140 |
-
|
141 |
-
def __len__(self):
|
142 |
-
return len(self.files)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / pymaf /utils /transforms.py
DELETED
@@ -1,78 +0,0 @@
|
|
1 |
-
# ------------------------------------------------------------------------------
|
2 |
-
# Copyright (c) Microsoft
|
3 |
-
# Licensed under the MIT License.
|
4 |
-
# Written by Bin Xiao ([email protected])
|
5 |
-
# ------------------------------------------------------------------------------
|
6 |
-
|
7 |
-
from __future__ import absolute_import
|
8 |
-
from __future__ import division
|
9 |
-
from __future__ import print_function
|
10 |
-
|
11 |
-
import cv2
|
12 |
-
import numpy as np
|
13 |
-
|
14 |
-
|
15 |
-
def transform_preds(coords, center, scale, output_size):
|
16 |
-
target_coords = np.zeros(coords.shape)
|
17 |
-
trans = get_affine_transform(center, scale, 0, output_size, inv=1)
|
18 |
-
for p in range(coords.shape[0]):
|
19 |
-
target_coords[p, 0:2] = affine_transform(coords[p, 0:2], trans)
|
20 |
-
return target_coords
|
21 |
-
|
22 |
-
|
23 |
-
def get_affine_transform(center,
|
24 |
-
scale,
|
25 |
-
rot,
|
26 |
-
output_size,
|
27 |
-
shift=np.array([0, 0], dtype=np.float32),
|
28 |
-
inv=0):
|
29 |
-
if not isinstance(scale, np.ndarray) and not isinstance(scale, list):
|
30 |
-
# print(scale)
|
31 |
-
scale = np.array([scale, scale])
|
32 |
-
|
33 |
-
scale_tmp = scale * 200.0
|
34 |
-
src_w = scale_tmp[0]
|
35 |
-
dst_w = output_size[0]
|
36 |
-
dst_h = output_size[1]
|
37 |
-
|
38 |
-
rot_rad = np.pi * rot / 180
|
39 |
-
src_dir = get_dir([0, src_w * -0.5], rot_rad)
|
40 |
-
dst_dir = np.array([0, dst_w * -0.5], np.float32)
|
41 |
-
|
42 |
-
src = np.zeros((3, 2), dtype=np.float32)
|
43 |
-
dst = np.zeros((3, 2), dtype=np.float32)
|
44 |
-
src[0, :] = center + scale_tmp * shift
|
45 |
-
src[1, :] = center + src_dir + scale_tmp * shift
|
46 |
-
dst[0, :] = [dst_w * 0.5, dst_h * 0.5]
|
47 |
-
dst[1, :] = np.array([dst_w * 0.5, dst_h * 0.5]) + dst_dir
|
48 |
-
|
49 |
-
src[2:, :] = get_3rd_point(src[0, :], src[1, :])
|
50 |
-
dst[2:, :] = get_3rd_point(dst[0, :], dst[1, :])
|
51 |
-
|
52 |
-
if inv:
|
53 |
-
trans = cv2.getAffineTransform(np.float32(dst), np.float32(src))
|
54 |
-
else:
|
55 |
-
trans = cv2.getAffineTransform(np.float32(src), np.float32(dst))
|
56 |
-
|
57 |
-
return trans
|
58 |
-
|
59 |
-
|
60 |
-
def affine_transform(pt, t):
|
61 |
-
new_pt = np.array([pt[0], pt[1], 1.]).T
|
62 |
-
new_pt = np.dot(t, new_pt)
|
63 |
-
return new_pt[:2]
|
64 |
-
|
65 |
-
|
66 |
-
def get_3rd_point(a, b):
|
67 |
-
direct = a - b
|
68 |
-
return b + np.array([-direct[1], direct[0]], dtype=np.float32)
|
69 |
-
|
70 |
-
|
71 |
-
def get_dir(src_point, rot_rad):
|
72 |
-
sn, cs = np.sin(rot_rad), np.cos(rot_rad)
|
73 |
-
|
74 |
-
src_result = [0, 0]
|
75 |
-
src_result[0] = src_point[0] * cs - src_point[1] * sn
|
76 |
-
src_result[1] = src_point[0] * sn + src_point[1] * cs
|
77 |
-
|
78 |
-
return src_result
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / __init__.py
DELETED
File without changes
|
lib / renderer / camera.py
DELETED
@@ -1,226 +0,0 @@
|
|
1 |
-
|
2 |
-
# -*- coding: utf-8 -*-
|
3 |
-
|
4 |
-
# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
|
5 |
-
# holder of all proprietary rights on this computer program.
|
6 |
-
# You can only use this computer program if you have closed
|
7 |
-
# a license agreement with MPG or you get the right to use the computer
|
8 |
-
# program from someone who is authorized to grant you that right.
|
9 |
-
# Any use of the computer program without a valid license is prohibited and
|
10 |
-
# liable to prosecution.
|
11 |
-
#
|
12 |
-
# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
|
13 |
-
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
|
14 |
-
# for Intelligent Systems. All rights reserved.
|
15 |
-
#
|
16 |
-
# Contact: [email protected]
|
17 |
-
|
18 |
-
import cv2
|
19 |
-
import numpy as np
|
20 |
-
|
21 |
-
from .glm import ortho
|
22 |
-
|
23 |
-
|
24 |
-
class Camera:
|
25 |
-
def __init__(self, width=1600, height=1200):
|
26 |
-
# Focal Length
|
27 |
-
# equivalent 50mm
|
28 |
-
focal = np.sqrt(width * width + height * height)
|
29 |
-
self.focal_x = focal
|
30 |
-
self.focal_y = focal
|
31 |
-
# Principal Point Offset
|
32 |
-
self.principal_x = width / 2
|
33 |
-
self.principal_y = height / 2
|
34 |
-
# Axis Skew
|
35 |
-
self.skew = 0
|
36 |
-
# Image Size
|
37 |
-
self.width = width
|
38 |
-
self.height = height
|
39 |
-
|
40 |
-
self.near = 1
|
41 |
-
self.far = 10
|
42 |
-
|
43 |
-
# Camera Center
|
44 |
-
self.center = np.array([0, 0, 1.6])
|
45 |
-
self.direction = np.array([0, 0, -1])
|
46 |
-
self.right = np.array([1, 0, 0])
|
47 |
-
self.up = np.array([0, 1, 0])
|
48 |
-
|
49 |
-
self.ortho_ratio = None
|
50 |
-
|
51 |
-
def sanity_check(self):
|
52 |
-
self.center = self.center.reshape([-1])
|
53 |
-
self.direction = self.direction.reshape([-1])
|
54 |
-
self.right = self.right.reshape([-1])
|
55 |
-
self.up = self.up.reshape([-1])
|
56 |
-
|
57 |
-
assert len(self.center) == 3
|
58 |
-
assert len(self.direction) == 3
|
59 |
-
assert len(self.right) == 3
|
60 |
-
assert len(self.up) == 3
|
61 |
-
|
62 |
-
@staticmethod
|
63 |
-
def normalize_vector(v):
|
64 |
-
v_norm = np.linalg.norm(v)
|
65 |
-
return v if v_norm == 0 else v / v_norm
|
66 |
-
|
67 |
-
def get_real_z_value(self, z):
|
68 |
-
z_near = self.near
|
69 |
-
z_far = self.far
|
70 |
-
z_n = 2.0 * z - 1.0
|
71 |
-
z_e = 2.0 * z_near * z_far / (z_far + z_near - z_n * (z_far - z_near))
|
72 |
-
return z_e
|
73 |
-
|
74 |
-
def get_rotation_matrix(self):
|
75 |
-
rot_mat = np.eye(3)
|
76 |
-
s = self.right
|
77 |
-
s = self.normalize_vector(s)
|
78 |
-
rot_mat[0, :] = s
|
79 |
-
u = self.up
|
80 |
-
u = self.normalize_vector(u)
|
81 |
-
rot_mat[1, :] = -u
|
82 |
-
rot_mat[2, :] = self.normalize_vector(self.direction)
|
83 |
-
|
84 |
-
return rot_mat
|
85 |
-
|
86 |
-
def get_translation_vector(self):
|
87 |
-
rot_mat = self.get_rotation_matrix()
|
88 |
-
trans = -np.dot(rot_mat, self.center)
|
89 |
-
return trans
|
90 |
-
|
91 |
-
def get_intrinsic_matrix(self):
|
92 |
-
int_mat = np.eye(3)
|
93 |
-
|
94 |
-
int_mat[0, 0] = self.focal_x
|
95 |
-
int_mat[1, 1] = self.focal_y
|
96 |
-
int_mat[0, 1] = self.skew
|
97 |
-
int_mat[0, 2] = self.principal_x
|
98 |
-
int_mat[1, 2] = self.principal_y
|
99 |
-
|
100 |
-
return int_mat
|
101 |
-
|
102 |
-
def get_projection_matrix(self):
|
103 |
-
ext_mat = self.get_extrinsic_matrix()
|
104 |
-
int_mat = self.get_intrinsic_matrix()
|
105 |
-
|
106 |
-
return np.matmul(int_mat, ext_mat)
|
107 |
-
|
108 |
-
def get_extrinsic_matrix(self):
|
109 |
-
rot_mat = self.get_rotation_matrix()
|
110 |
-
int_mat = self.get_intrinsic_matrix()
|
111 |
-
trans = self.get_translation_vector()
|
112 |
-
|
113 |
-
extrinsic = np.eye(4)
|
114 |
-
extrinsic[:3, :3] = rot_mat
|
115 |
-
extrinsic[:3, 3] = trans
|
116 |
-
|
117 |
-
return extrinsic[:3, :]
|
118 |
-
|
119 |
-
def set_rotation_matrix(self, rot_mat):
|
120 |
-
self.direction = rot_mat[2, :]
|
121 |
-
self.up = -rot_mat[1, :]
|
122 |
-
self.right = rot_mat[0, :]
|
123 |
-
|
124 |
-
def set_intrinsic_matrix(self, int_mat):
|
125 |
-
self.focal_x = int_mat[0, 0]
|
126 |
-
self.focal_y = int_mat[1, 1]
|
127 |
-
self.skew = int_mat[0, 1]
|
128 |
-
self.principal_x = int_mat[0, 2]
|
129 |
-
self.principal_y = int_mat[1, 2]
|
130 |
-
|
131 |
-
def set_projection_matrix(self, proj_mat):
|
132 |
-
res = cv2.decomposeProjectionMatrix(proj_mat)
|
133 |
-
int_mat, rot_mat, camera_center_homo = res[0], res[1], res[2]
|
134 |
-
camera_center = camera_center_homo[0:3] / camera_center_homo[3]
|
135 |
-
camera_center = camera_center.reshape(-1)
|
136 |
-
int_mat = int_mat / int_mat[2][2]
|
137 |
-
|
138 |
-
self.set_intrinsic_matrix(int_mat)
|
139 |
-
self.set_rotation_matrix(rot_mat)
|
140 |
-
self.center = camera_center
|
141 |
-
|
142 |
-
self.sanity_check()
|
143 |
-
|
144 |
-
def get_gl_matrix(self):
|
145 |
-
z_near = self.near
|
146 |
-
z_far = self.far
|
147 |
-
rot_mat = self.get_rotation_matrix()
|
148 |
-
int_mat = self.get_intrinsic_matrix()
|
149 |
-
trans = self.get_translation_vector()
|
150 |
-
|
151 |
-
extrinsic = np.eye(4)
|
152 |
-
extrinsic[:3, :3] = rot_mat
|
153 |
-
extrinsic[:3, 3] = trans
|
154 |
-
axis_adj = np.eye(4)
|
155 |
-
axis_adj[2, 2] = -1
|
156 |
-
axis_adj[1, 1] = -1
|
157 |
-
model_view = np.matmul(axis_adj, extrinsic)
|
158 |
-
|
159 |
-
projective = np.zeros([4, 4])
|
160 |
-
projective[:2, :2] = int_mat[:2, :2]
|
161 |
-
projective[:2, 2:3] = -int_mat[:2, 2:3]
|
162 |
-
projective[3, 2] = -1
|
163 |
-
projective[2, 2] = (z_near + z_far)
|
164 |
-
projective[2, 3] = (z_near * z_far)
|
165 |
-
|
166 |
-
if self.ortho_ratio is None:
|
167 |
-
ndc = ortho(0, self.width, 0, self.height, z_near, z_far)
|
168 |
-
perspective = np.matmul(ndc, projective)
|
169 |
-
else:
|
170 |
-
perspective = ortho(-self.width * self.ortho_ratio / 2,
|
171 |
-
self.width * self.ortho_ratio / 2,
|
172 |
-
-self.height * self.ortho_ratio / 2,
|
173 |
-
self.height * self.ortho_ratio / 2, z_near,
|
174 |
-
z_far)
|
175 |
-
|
176 |
-
return perspective, model_view
|
177 |
-
|
178 |
-
|
179 |
-
def KRT_from_P(proj_mat, normalize_K=True):
|
180 |
-
res = cv2.decomposeProjectionMatrix(proj_mat)
|
181 |
-
K, Rot, camera_center_homog = res[0], res[1], res[2]
|
182 |
-
camera_center = camera_center_homog[0:3] / camera_center_homog[3]
|
183 |
-
trans = -Rot.dot(camera_center)
|
184 |
-
if normalize_K:
|
185 |
-
K = K / K[2][2]
|
186 |
-
return K, Rot, trans
|
187 |
-
|
188 |
-
|
189 |
-
def MVP_from_P(proj_mat, width, height, near=0.1, far=10000):
|
190 |
-
'''
|
191 |
-
Convert OpenCV camera calibration matrix to OpenGL projection and model view matrix
|
192 |
-
:param proj_mat: OpenCV camera projeciton matrix
|
193 |
-
:param width: Image width
|
194 |
-
:param height: Image height
|
195 |
-
:param near: Z near value
|
196 |
-
:param far: Z far value
|
197 |
-
:return: OpenGL projection matrix and model view matrix
|
198 |
-
'''
|
199 |
-
res = cv2.decomposeProjectionMatrix(proj_mat)
|
200 |
-
K, Rot, camera_center_homog = res[0], res[1], res[2]
|
201 |
-
camera_center = camera_center_homog[0:3] / camera_center_homog[3]
|
202 |
-
trans = -Rot.dot(camera_center)
|
203 |
-
K = K / K[2][2]
|
204 |
-
|
205 |
-
extrinsic = np.eye(4)
|
206 |
-
extrinsic[:3, :3] = Rot
|
207 |
-
extrinsic[:3, 3:4] = trans
|
208 |
-
axis_adj = np.eye(4)
|
209 |
-
axis_adj[2, 2] = -1
|
210 |
-
axis_adj[1, 1] = -1
|
211 |
-
model_view = np.matmul(axis_adj, extrinsic)
|
212 |
-
|
213 |
-
zFar = far
|
214 |
-
zNear = near
|
215 |
-
projective = np.zeros([4, 4])
|
216 |
-
projective[:2, :2] = K[:2, :2]
|
217 |
-
projective[:2, 2:3] = -K[:2, 2:3]
|
218 |
-
projective[3, 2] = -1
|
219 |
-
projective[2, 2] = (zNear + zFar)
|
220 |
-
projective[2, 3] = (zNear * zFar)
|
221 |
-
|
222 |
-
ndc = ortho(0, width, 0, height, zNear, zFar)
|
223 |
-
|
224 |
-
perspective = np.matmul(ndc, projective)
|
225 |
-
|
226 |
-
return perspective, model_view
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / __init__.py
DELETED
File without changes
|
lib / renderer / gl / data / color.fs
DELETED
@@ -1,20 +0,0 @@
|
|
1 |
-
#version 330 core
|
2 |
-
|
3 |
-
layout (location = 0) out vec4 FragColor;
|
4 |
-
layout (location = 1) out vec4 FragNormal;
|
5 |
-
layout (location = 2) out vec4 FragDepth;
|
6 |
-
|
7 |
-
in vec3 Color;
|
8 |
-
in vec3 CamNormal;
|
9 |
-
in vec3 depth;
|
10 |
-
|
11 |
-
|
12 |
-
void main()
|
13 |
-
{
|
14 |
-
FragColor = vec4(Color,1.0);
|
15 |
-
|
16 |
-
vec3 cam_norm_normalized = normalize(CamNormal);
|
17 |
-
vec3 rgb = (cam_norm_normalized + 1.0) / 2.0;
|
18 |
-
FragNormal = vec4(rgb, 1.0);
|
19 |
-
FragDepth = vec4(depth.xyz, 1.0);
|
20 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / data /color.vs
DELETED
@@ -1,29 +0,0 @@
|
|
1 |
-
#version 330 core
|
2 |
-
|
3 |
-
layout (location = 0) in vec3 a_Position;
|
4 |
-
layout (location = 1) in vec3 a_Color;
|
5 |
-
layout (location = 2) in vec3 a_Normal;
|
6 |
-
|
7 |
-
out vec3 CamNormal;
|
8 |
-
out vec3 CamPos;
|
9 |
-
out vec3 Color;
|
10 |
-
out vec3 depth;
|
11 |
-
|
12 |
-
|
13 |
-
uniform mat3 RotMat;
|
14 |
-
uniform mat4 NormMat;
|
15 |
-
uniform mat4 ModelMat;
|
16 |
-
uniform mat4 PerspMat;
|
17 |
-
|
18 |
-
void main()
|
19 |
-
{
|
20 |
-
vec3 a_Position = (NormMat * vec4(a_Position,1.0)).xyz;
|
21 |
-
gl_Position = PerspMat * ModelMat * vec4(RotMat * a_Position, 1.0);
|
22 |
-
Color = a_Color;
|
23 |
-
|
24 |
-
mat3 R = mat3(ModelMat) * RotMat;
|
25 |
-
CamNormal = (R * a_Normal);
|
26 |
-
|
27 |
-
depth = vec3(gl_Position.z / gl_Position.w);
|
28 |
-
|
29 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / data /normal.fs
DELETED
@@ -1,12 +0,0 @@
|
|
1 |
-
#version 330
|
2 |
-
|
3 |
-
out vec4 FragColor;
|
4 |
-
|
5 |
-
in vec3 CamNormal;
|
6 |
-
|
7 |
-
void main()
|
8 |
-
{
|
9 |
-
vec3 cam_norm_normalized = normalize(CamNormal);
|
10 |
-
vec3 rgb = (cam_norm_normalized + 1.0) / 2.0;
|
11 |
-
FragColor = vec4(rgb, 1.0);
|
12 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / data /normal.vs
DELETED
@@ -1,15 +0,0 @@
|
|
1 |
-
#version 330
|
2 |
-
|
3 |
-
layout (location = 0) in vec3 Position;
|
4 |
-
layout (location = 1) in vec3 Normal;
|
5 |
-
|
6 |
-
out vec3 CamNormal;
|
7 |
-
|
8 |
-
uniform mat4 ModelMat;
|
9 |
-
uniform mat4 PerspMat;
|
10 |
-
|
11 |
-
void main()
|
12 |
-
{
|
13 |
-
gl_Position = PerspMat * ModelMat * vec4(Position, 1.0);
|
14 |
-
CamNormal = (ModelMat * vec4(Normal, 0.0)).xyz;
|
15 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / data /prt.fs
DELETED
@@ -1,157 +0,0 @@
|
|
1 |
-
#version 330
|
2 |
-
|
3 |
-
uniform vec3 SHCoeffs[9];
|
4 |
-
uniform uint analytic;
|
5 |
-
|
6 |
-
uniform uint hasNormalMap;
|
7 |
-
uniform uint hasAlbedoMap;
|
8 |
-
|
9 |
-
uniform sampler2D AlbedoMap;
|
10 |
-
uniform sampler2D NormalMap;
|
11 |
-
|
12 |
-
in VertexData {
|
13 |
-
vec3 Position;
|
14 |
-
vec3 Depth;
|
15 |
-
vec3 ModelNormal;
|
16 |
-
vec2 Texcoord;
|
17 |
-
vec3 Tangent;
|
18 |
-
vec3 Bitangent;
|
19 |
-
vec3 PRT1;
|
20 |
-
vec3 PRT2;
|
21 |
-
vec3 PRT3;
|
22 |
-
vec3 Label;
|
23 |
-
} VertexIn;
|
24 |
-
|
25 |
-
layout (location = 0) out vec4 FragColor;
|
26 |
-
layout (location = 1) out vec4 FragNormal;
|
27 |
-
layout (location = 2) out vec4 FragPosition;
|
28 |
-
layout (location = 3) out vec4 FragAlbedo;
|
29 |
-
layout (location = 4) out vec4 FragShading;
|
30 |
-
layout (location = 5) out vec4 FragPRT1;
|
31 |
-
layout (location = 6) out vec4 FragPRT2;
|
32 |
-
// layout (location = 7) out vec4 FragPRT3;
|
33 |
-
layout (location = 7) out vec4 FragLabel;
|
34 |
-
|
35 |
-
|
36 |
-
vec4 gammaCorrection(vec4 vec, float g)
|
37 |
-
{
|
38 |
-
return vec4(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g), vec.w);
|
39 |
-
}
|
40 |
-
|
41 |
-
vec3 gammaCorrection(vec3 vec, float g)
|
42 |
-
{
|
43 |
-
return vec3(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g));
|
44 |
-
}
|
45 |
-
|
46 |
-
void evaluateH(vec3 n, out float H[9])
|
47 |
-
{
|
48 |
-
float c1 = 0.429043, c2 = 0.511664,
|
49 |
-
c3 = 0.743125, c4 = 0.886227, c5 = 0.247708;
|
50 |
-
|
51 |
-
H[0] = c4;
|
52 |
-
H[1] = 2.0 * c2 * n[1];
|
53 |
-
H[2] = 2.0 * c2 * n[2];
|
54 |
-
H[3] = 2.0 * c2 * n[0];
|
55 |
-
H[4] = 2.0 * c1 * n[0] * n[1];
|
56 |
-
H[5] = 2.0 * c1 * n[1] * n[2];
|
57 |
-
H[6] = c3 * n[2] * n[2] - c5;
|
58 |
-
H[7] = 2.0 * c1 * n[2] * n[0];
|
59 |
-
H[8] = c1 * (n[0] * n[0] - n[1] * n[1]);
|
60 |
-
}
|
61 |
-
|
62 |
-
vec3 evaluateLightingModel(vec3 normal)
|
63 |
-
{
|
64 |
-
float H[9];
|
65 |
-
evaluateH(normal, H);
|
66 |
-
vec3 res = vec3(0.0);
|
67 |
-
for (int i = 0; i < 9; i++) {
|
68 |
-
res += H[i] * SHCoeffs[i];
|
69 |
-
}
|
70 |
-
return res;
|
71 |
-
}
|
72 |
-
|
73 |
-
// nC: coarse geometry normal, nH: fine normal from normal map
|
74 |
-
vec3 evaluateLightingModelHybrid(vec3 nC, vec3 nH, mat3 prt)
|
75 |
-
{
|
76 |
-
float HC[9], HH[9];
|
77 |
-
evaluateH(nC, HC);
|
78 |
-
evaluateH(nH, HH);
|
79 |
-
|
80 |
-
vec3 res = vec3(0.0);
|
81 |
-
vec3 shadow = vec3(0.0);
|
82 |
-
vec3 unshadow = vec3(0.0);
|
83 |
-
for(int i = 0; i < 3; ++i){
|
84 |
-
for(int j = 0; j < 3; ++j){
|
85 |
-
int id = i*3+j;
|
86 |
-
res += HH[id]* SHCoeffs[id];
|
87 |
-
shadow += prt[i][j] * SHCoeffs[id];
|
88 |
-
unshadow += HC[id] * SHCoeffs[id];
|
89 |
-
}
|
90 |
-
}
|
91 |
-
vec3 ratio = clamp(shadow/unshadow,0.0,1.0);
|
92 |
-
res = ratio * res;
|
93 |
-
|
94 |
-
return res;
|
95 |
-
}
|
96 |
-
|
97 |
-
vec3 evaluateLightingModelPRT(mat3 prt)
|
98 |
-
{
|
99 |
-
vec3 res = vec3(0.0);
|
100 |
-
for(int i = 0; i < 3; ++i){
|
101 |
-
for(int j = 0; j < 3; ++j){
|
102 |
-
res += prt[i][j] * SHCoeffs[i*3+j];
|
103 |
-
}
|
104 |
-
}
|
105 |
-
|
106 |
-
return res;
|
107 |
-
}
|
108 |
-
|
109 |
-
void main()
|
110 |
-
{
|
111 |
-
vec2 uv = VertexIn.Texcoord;
|
112 |
-
vec3 nC = normalize(VertexIn.ModelNormal);
|
113 |
-
vec3 nml = nC;
|
114 |
-
mat3 prt = mat3(VertexIn.PRT1, VertexIn.PRT2, VertexIn.PRT3);
|
115 |
-
|
116 |
-
if(hasAlbedoMap == uint(0))
|
117 |
-
FragAlbedo = vec4(1.0);
|
118 |
-
else
|
119 |
-
FragAlbedo = texture(AlbedoMap, uv);//gammaCorrection(texture(AlbedoMap, uv), 1.0/2.2);
|
120 |
-
|
121 |
-
if(hasNormalMap == uint(0))
|
122 |
-
{
|
123 |
-
if(analytic == uint(0))
|
124 |
-
FragShading = vec4(evaluateLightingModelPRT(prt), 1.0f);
|
125 |
-
else
|
126 |
-
FragShading = vec4(evaluateLightingModel(nC), 1.0f);
|
127 |
-
}
|
128 |
-
else
|
129 |
-
{
|
130 |
-
vec3 n_tan = normalize(texture(NormalMap, uv).rgb*2.0-vec3(1.0));
|
131 |
-
|
132 |
-
mat3 TBN = mat3(normalize(VertexIn.Tangent),normalize(VertexIn.Bitangent),nC);
|
133 |
-
vec3 nH = normalize(TBN * n_tan);
|
134 |
-
|
135 |
-
if(analytic == uint(0))
|
136 |
-
FragShading = vec4(evaluateLightingModelHybrid(nC,nH,prt),1.0f);
|
137 |
-
else
|
138 |
-
FragShading = vec4(evaluateLightingModel(nH), 1.0f);
|
139 |
-
|
140 |
-
nml = nH;
|
141 |
-
}
|
142 |
-
|
143 |
-
FragShading = gammaCorrection(FragShading, 2.2);
|
144 |
-
FragColor = clamp(FragAlbedo * FragShading, 0.0, 1.0);
|
145 |
-
FragNormal = vec4(0.5*(nml+vec3(1.0)), 1.0);
|
146 |
-
FragPosition = vec4(VertexIn.Depth.xyz, 1.0);
|
147 |
-
FragShading = vec4(clamp(0.5*FragShading.xyz, 0.0, 1.0),1.0);
|
148 |
-
// FragColor = gammaCorrection(clamp(FragAlbedo * FragShading, 0.0, 1.0),2.2);
|
149 |
-
// FragNormal = vec4(0.5*(nml+vec3(1.0)), 1.0);
|
150 |
-
// FragPosition = vec4(VertexIn.Position,VertexIn.Depth.x);
|
151 |
-
// FragShading = vec4(gammaCorrection(clamp(0.5*FragShading.xyz, 0.0, 1.0),2.2),1.0);
|
152 |
-
// FragAlbedo = gammaCorrection(FragAlbedo,2.2);
|
153 |
-
FragPRT1 = vec4(VertexIn.PRT1,1.0);
|
154 |
-
FragPRT2 = vec4(VertexIn.PRT2,1.0);
|
155 |
-
// FragPRT3 = vec4(VertexIn.PRT3,1.0);
|
156 |
-
FragLabel = vec4(VertexIn.Label,1.0);
|
157 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / data /prt.vs
DELETED
@@ -1,156 +0,0 @@
|
|
1 |
-
#version 330
|
2 |
-
|
3 |
-
layout (location = 0) in vec3 a_Position;
|
4 |
-
layout (location = 1) in vec3 a_Normal;
|
5 |
-
layout (location = 2) in vec2 a_TextureCoord;
|
6 |
-
layout (location = 3) in vec3 a_Tangent;
|
7 |
-
layout (location = 4) in vec3 a_Bitangent;
|
8 |
-
layout (location = 5) in vec3 a_PRT1;
|
9 |
-
layout (location = 6) in vec3 a_PRT2;
|
10 |
-
layout (location = 7) in vec3 a_PRT3;
|
11 |
-
layout (location = 8) in vec3 a_Label;
|
12 |
-
out VertexData {
|
13 |
-
vec3 Position;
|
14 |
-
vec3 Depth;
|
15 |
-
vec3 ModelNormal;
|
16 |
-
vec2 Texcoord;
|
17 |
-
vec3 Tangent;
|
18 |
-
vec3 Bitangent;
|
19 |
-
vec3 PRT1;
|
20 |
-
vec3 PRT2;
|
21 |
-
vec3 PRT3;
|
22 |
-
vec3 Label;
|
23 |
-
} VertexOut;
|
24 |
-
uniform mat3 RotMat;
|
25 |
-
uniform mat4 NormMat;
|
26 |
-
uniform mat4 ModelMat;
|
27 |
-
uniform mat4 PerspMat;
|
28 |
-
float s_c3 = 0.94617469575; // (3*sqrt(5))/(4*sqrt(pi))
|
29 |
-
float s_c4 = -0.31539156525;// (-sqrt(5))/(4*sqrt(pi))
|
30 |
-
float s_c5 = 0.54627421529; // (sqrt(15))/(4*sqrt(pi))
|
31 |
-
float s_c_scale = 1.0/0.91529123286551084;
|
32 |
-
float s_c_scale_inv = 0.91529123286551084;
|
33 |
-
float s_rc2 = 1.5853309190550713*s_c_scale;
|
34 |
-
float s_c4_div_c3 = s_c4/s_c3;
|
35 |
-
float s_c4_div_c3_x2 = (s_c4/s_c3)*2.0;
|
36 |
-
float s_scale_dst2 = s_c3 * s_c_scale_inv;
|
37 |
-
float s_scale_dst4 = s_c5 * s_c_scale_inv;
|
38 |
-
void OptRotateBand0(float x[1], mat3 R, out float dst[1])
|
39 |
-
{
|
40 |
-
dst[0] = x[0];
|
41 |
-
}
|
42 |
-
// 9 multiplies
|
43 |
-
void OptRotateBand1(float x[3], mat3 R, out float dst[3])
|
44 |
-
{
|
45 |
-
// derived from SlowRotateBand1
|
46 |
-
dst[0] = ( R[1][1])*x[0] + (-R[1][2])*x[1] + ( R[1][0])*x[2];
|
47 |
-
dst[1] = (-R[2][1])*x[0] + ( R[2][2])*x[1] + (-R[2][0])*x[2];
|
48 |
-
dst[2] = ( R[0][1])*x[0] + (-R[0][2])*x[1] + ( R[0][0])*x[2];
|
49 |
-
}
|
50 |
-
// 48 multiplies
|
51 |
-
void OptRotateBand2(float x[5], mat3 R, out float dst[5])
|
52 |
-
{
|
53 |
-
// Sparse matrix multiply
|
54 |
-
float sh0 = x[3] + x[4] + x[4] - x[1];
|
55 |
-
float sh1 = x[0] + s_rc2*x[2] + x[3] + x[4];
|
56 |
-
float sh2 = x[0];
|
57 |
-
float sh3 = -x[3];
|
58 |
-
float sh4 = -x[1];
|
59 |
-
|
60 |
-
// Rotations. R0 and R1 just use the raw matrix columns
|
61 |
-
float r2x = R[0][0] + R[0][1];
|
62 |
-
float r2y = R[1][0] + R[1][1];
|
63 |
-
float r2z = R[2][0] + R[2][1];
|
64 |
-
|
65 |
-
float r3x = R[0][0] + R[0][2];
|
66 |
-
float r3y = R[1][0] + R[1][2];
|
67 |
-
float r3z = R[2][0] + R[2][2];
|
68 |
-
|
69 |
-
float r4x = R[0][1] + R[0][2];
|
70 |
-
float r4y = R[1][1] + R[1][2];
|
71 |
-
float r4z = R[2][1] + R[2][2];
|
72 |
-
|
73 |
-
// dense matrix multiplication one column at a time
|
74 |
-
|
75 |
-
// column 0
|
76 |
-
float sh0_x = sh0 * R[0][0];
|
77 |
-
float sh0_y = sh0 * R[1][0];
|
78 |
-
float d0 = sh0_x * R[1][0];
|
79 |
-
float d1 = sh0_y * R[2][0];
|
80 |
-
float d2 = sh0 * (R[2][0] * R[2][0] + s_c4_div_c3);
|
81 |
-
float d3 = sh0_x * R[2][0];
|
82 |
-
float d4 = sh0_x * R[0][0] - sh0_y * R[1][0];
|
83 |
-
|
84 |
-
// column 1
|
85 |
-
float sh1_x = sh1 * R[0][2];
|
86 |
-
float sh1_y = sh1 * R[1][2];
|
87 |
-
d0 += sh1_x * R[1][2];
|
88 |
-
d1 += sh1_y * R[2][2];
|
89 |
-
d2 += sh1 * (R[2][2] * R[2][2] + s_c4_div_c3);
|
90 |
-
d3 += sh1_x * R[2][2];
|
91 |
-
d4 += sh1_x * R[0][2] - sh1_y * R[1][2];
|
92 |
-
|
93 |
-
// column 2
|
94 |
-
float sh2_x = sh2 * r2x;
|
95 |
-
float sh2_y = sh2 * r2y;
|
96 |
-
d0 += sh2_x * r2y;
|
97 |
-
d1 += sh2_y * r2z;
|
98 |
-
d2 += sh2 * (r2z * r2z + s_c4_div_c3_x2);
|
99 |
-
d3 += sh2_x * r2z;
|
100 |
-
d4 += sh2_x * r2x - sh2_y * r2y;
|
101 |
-
|
102 |
-
// column 3
|
103 |
-
float sh3_x = sh3 * r3x;
|
104 |
-
float sh3_y = sh3 * r3y;
|
105 |
-
d0 += sh3_x * r3y;
|
106 |
-
d1 += sh3_y * r3z;
|
107 |
-
d2 += sh3 * (r3z * r3z + s_c4_div_c3_x2);
|
108 |
-
d3 += sh3_x * r3z;
|
109 |
-
d4 += sh3_x * r3x - sh3_y * r3y;
|
110 |
-
|
111 |
-
// column 4
|
112 |
-
float sh4_x = sh4 * r4x;
|
113 |
-
float sh4_y = sh4 * r4y;
|
114 |
-
d0 += sh4_x * r4y;
|
115 |
-
d1 += sh4_y * r4z;
|
116 |
-
d2 += sh4 * (r4z * r4z + s_c4_div_c3_x2);
|
117 |
-
d3 += sh4_x * r4z;
|
118 |
-
d4 += sh4_x * r4x - sh4_y * r4y;
|
119 |
-
|
120 |
-
// extra multipliers
|
121 |
-
dst[0] = d0;
|
122 |
-
dst[1] = -d1;
|
123 |
-
dst[2] = d2 * s_scale_dst2;
|
124 |
-
dst[3] = -d3;
|
125 |
-
dst[4] = d4 * s_scale_dst4;
|
126 |
-
}
|
127 |
-
void main()
|
128 |
-
{
|
129 |
-
// normalization
|
130 |
-
vec3 pos = (NormMat * vec4(a_Position,1.0)).xyz;
|
131 |
-
mat3 R = mat3(ModelMat) * RotMat;
|
132 |
-
VertexOut.ModelNormal = (R * a_Normal);
|
133 |
-
VertexOut.Position = R * pos;
|
134 |
-
VertexOut.Texcoord = a_TextureCoord;
|
135 |
-
VertexOut.Tangent = (R * a_Tangent);
|
136 |
-
VertexOut.Bitangent = (R * a_Bitangent);
|
137 |
-
VertexOut.Label = a_Label;
|
138 |
-
float PRT0, PRT1[3], PRT2[5];
|
139 |
-
PRT0 = a_PRT1[0];
|
140 |
-
PRT1[0] = a_PRT1[1];
|
141 |
-
PRT1[1] = a_PRT1[2];
|
142 |
-
PRT1[2] = a_PRT2[0];
|
143 |
-
PRT2[0] = a_PRT2[1];
|
144 |
-
PRT2[1] = a_PRT2[2];
|
145 |
-
PRT2[2] = a_PRT3[0];
|
146 |
-
PRT2[3] = a_PRT3[1];
|
147 |
-
PRT2[4] = a_PRT3[2];
|
148 |
-
OptRotateBand1(PRT1, R, PRT1);
|
149 |
-
OptRotateBand2(PRT2, R, PRT2);
|
150 |
-
VertexOut.PRT1 = vec3(PRT0,PRT1[0],PRT1[1]);
|
151 |
-
VertexOut.PRT2 = vec3(PRT1[2],PRT2[0],PRT2[1]);
|
152 |
-
VertexOut.PRT3 = vec3(PRT2[2],PRT2[3],PRT2[4]);
|
153 |
-
gl_Position = PerspMat * ModelMat * vec4(RotMat * pos, 1.0);
|
154 |
-
|
155 |
-
VertexOut.Depth = vec3(gl_Position.z / gl_Position.w);
|
156 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / data /prt_uv.fs
DELETED
@@ -1,141 +0,0 @@
|
|
1 |
-
#version 330
|
2 |
-
|
3 |
-
uniform vec3 SHCoeffs[9];
|
4 |
-
uniform uint analytic;
|
5 |
-
|
6 |
-
uniform uint hasNormalMap;
|
7 |
-
uniform uint hasAlbedoMap;
|
8 |
-
|
9 |
-
uniform sampler2D AlbedoMap;
|
10 |
-
uniform sampler2D NormalMap;
|
11 |
-
|
12 |
-
in VertexData {
|
13 |
-
vec3 Position;
|
14 |
-
vec3 ModelNormal;
|
15 |
-
vec3 CameraNormal;
|
16 |
-
vec2 Texcoord;
|
17 |
-
vec3 Tangent;
|
18 |
-
vec3 Bitangent;
|
19 |
-
vec3 PRT1;
|
20 |
-
vec3 PRT2;
|
21 |
-
vec3 PRT3;
|
22 |
-
} VertexIn;
|
23 |
-
|
24 |
-
layout (location = 0) out vec4 FragColor;
|
25 |
-
layout (location = 1) out vec4 FragPosition;
|
26 |
-
layout (location = 2) out vec4 FragNormal;
|
27 |
-
|
28 |
-
vec4 gammaCorrection(vec4 vec, float g)
|
29 |
-
{
|
30 |
-
return vec4(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g), vec.w);
|
31 |
-
}
|
32 |
-
|
33 |
-
vec3 gammaCorrection(vec3 vec, float g)
|
34 |
-
{
|
35 |
-
return vec3(pow(vec.x, 1.0/g), pow(vec.y, 1.0/g), pow(vec.z, 1.0/g));
|
36 |
-
}
|
37 |
-
|
38 |
-
void evaluateH(vec3 n, out float H[9])
|
39 |
-
{
|
40 |
-
float c1 = 0.429043, c2 = 0.511664,
|
41 |
-
c3 = 0.743125, c4 = 0.886227, c5 = 0.247708;
|
42 |
-
|
43 |
-
H[0] = c4;
|
44 |
-
H[1] = 2.0 * c2 * n[1];
|
45 |
-
H[2] = 2.0 * c2 * n[2];
|
46 |
-
H[3] = 2.0 * c2 * n[0];
|
47 |
-
H[4] = 2.0 * c1 * n[0] * n[1];
|
48 |
-
H[5] = 2.0 * c1 * n[1] * n[2];
|
49 |
-
H[6] = c3 * n[2] * n[2] - c5;
|
50 |
-
H[7] = 2.0 * c1 * n[2] * n[0];
|
51 |
-
H[8] = c1 * (n[0] * n[0] - n[1] * n[1]);
|
52 |
-
}
|
53 |
-
|
54 |
-
vec3 evaluateLightingModel(vec3 normal)
|
55 |
-
{
|
56 |
-
float H[9];
|
57 |
-
evaluateH(normal, H);
|
58 |
-
vec3 res = vec3(0.0);
|
59 |
-
for (int i = 0; i < 9; i++) {
|
60 |
-
res += H[i] * SHCoeffs[i];
|
61 |
-
}
|
62 |
-
return res;
|
63 |
-
}
|
64 |
-
|
65 |
-
// nC: coarse geometry normal, nH: fine normal from normal map
|
66 |
-
vec3 evaluateLightingModelHybrid(vec3 nC, vec3 nH, mat3 prt)
|
67 |
-
{
|
68 |
-
float HC[9], HH[9];
|
69 |
-
evaluateH(nC, HC);
|
70 |
-
evaluateH(nH, HH);
|
71 |
-
|
72 |
-
vec3 res = vec3(0.0);
|
73 |
-
vec3 shadow = vec3(0.0);
|
74 |
-
vec3 unshadow = vec3(0.0);
|
75 |
-
for(int i = 0; i < 3; ++i){
|
76 |
-
for(int j = 0; j < 3; ++j){
|
77 |
-
int id = i*3+j;
|
78 |
-
res += HH[id]* SHCoeffs[id];
|
79 |
-
shadow += prt[i][j] * SHCoeffs[id];
|
80 |
-
unshadow += HC[id] * SHCoeffs[id];
|
81 |
-
}
|
82 |
-
}
|
83 |
-
vec3 ratio = clamp(shadow/unshadow,0.0,1.0);
|
84 |
-
res = ratio * res;
|
85 |
-
|
86 |
-
return res;
|
87 |
-
}
|
88 |
-
|
89 |
-
vec3 evaluateLightingModelPRT(mat3 prt)
|
90 |
-
{
|
91 |
-
vec3 res = vec3(0.0);
|
92 |
-
for(int i = 0; i < 3; ++i){
|
93 |
-
for(int j = 0; j < 3; ++j){
|
94 |
-
res += prt[i][j] * SHCoeffs[i*3+j];
|
95 |
-
}
|
96 |
-
}
|
97 |
-
|
98 |
-
return res;
|
99 |
-
}
|
100 |
-
|
101 |
-
void main()
|
102 |
-
{
|
103 |
-
vec2 uv = VertexIn.Texcoord;
|
104 |
-
vec3 nM = normalize(VertexIn.ModelNormal);
|
105 |
-
vec3 nC = normalize(VertexIn.CameraNormal);
|
106 |
-
vec3 nml = nC;
|
107 |
-
mat3 prt = mat3(VertexIn.PRT1, VertexIn.PRT2, VertexIn.PRT3);
|
108 |
-
|
109 |
-
vec4 albedo, shading;
|
110 |
-
if(hasAlbedoMap == uint(0))
|
111 |
-
albedo = vec4(1.0);
|
112 |
-
else
|
113 |
-
albedo = texture(AlbedoMap, uv);//gammaCorrection(texture(AlbedoMap, uv), 1.0/2.2);
|
114 |
-
|
115 |
-
if(hasNormalMap == uint(0))
|
116 |
-
{
|
117 |
-
if(analytic == uint(0))
|
118 |
-
shading = vec4(evaluateLightingModelPRT(prt), 1.0f);
|
119 |
-
else
|
120 |
-
shading = vec4(evaluateLightingModel(nC), 1.0f);
|
121 |
-
}
|
122 |
-
else
|
123 |
-
{
|
124 |
-
vec3 n_tan = normalize(texture(NormalMap, uv).rgb*2.0-vec3(1.0));
|
125 |
-
|
126 |
-
mat3 TBN = mat3(normalize(VertexIn.Tangent),normalize(VertexIn.Bitangent),nC);
|
127 |
-
vec3 nH = normalize(TBN * n_tan);
|
128 |
-
|
129 |
-
if(analytic == uint(0))
|
130 |
-
shading = vec4(evaluateLightingModelHybrid(nC,nH,prt),1.0f);
|
131 |
-
else
|
132 |
-
shading = vec4(evaluateLightingModel(nH), 1.0f);
|
133 |
-
|
134 |
-
nml = nH;
|
135 |
-
}
|
136 |
-
|
137 |
-
shading = gammaCorrection(shading, 2.2);
|
138 |
-
FragColor = clamp(albedo * shading, 0.0, 1.0);
|
139 |
-
FragPosition = vec4(VertexIn.Position,1.0);
|
140 |
-
FragNormal = vec4(0.5*(nM+vec3(1.0)),1.0);
|
141 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / data /prt_uv.vs
DELETED
@@ -1,168 +0,0 @@
|
|
1 |
-
#version 330
|
2 |
-
|
3 |
-
layout (location = 0) in vec3 a_Position;
|
4 |
-
layout (location = 1) in vec3 a_Normal;
|
5 |
-
layout (location = 2) in vec2 a_TextureCoord;
|
6 |
-
layout (location = 3) in vec3 a_Tangent;
|
7 |
-
layout (location = 4) in vec3 a_Bitangent;
|
8 |
-
layout (location = 5) in vec3 a_PRT1;
|
9 |
-
layout (location = 6) in vec3 a_PRT2;
|
10 |
-
layout (location = 7) in vec3 a_PRT3;
|
11 |
-
|
12 |
-
out VertexData {
|
13 |
-
vec3 Position;
|
14 |
-
vec3 ModelNormal;
|
15 |
-
vec3 CameraNormal;
|
16 |
-
vec2 Texcoord;
|
17 |
-
vec3 Tangent;
|
18 |
-
vec3 Bitangent;
|
19 |
-
vec3 PRT1;
|
20 |
-
vec3 PRT2;
|
21 |
-
vec3 PRT3;
|
22 |
-
} VertexOut;
|
23 |
-
|
24 |
-
uniform mat3 RotMat;
|
25 |
-
uniform mat4 NormMat;
|
26 |
-
uniform mat4 ModelMat;
|
27 |
-
uniform mat4 PerspMat;
|
28 |
-
|
29 |
-
#define pi 3.1415926535897932384626433832795
|
30 |
-
|
31 |
-
float s_c3 = 0.94617469575; // (3*sqrt(5))/(4*sqrt(pi))
|
32 |
-
float s_c4 = -0.31539156525;// (-sqrt(5))/(4*sqrt(pi))
|
33 |
-
float s_c5 = 0.54627421529; // (sqrt(15))/(4*sqrt(pi))
|
34 |
-
|
35 |
-
float s_c_scale = 1.0/0.91529123286551084;
|
36 |
-
float s_c_scale_inv = 0.91529123286551084;
|
37 |
-
|
38 |
-
float s_rc2 = 1.5853309190550713*s_c_scale;
|
39 |
-
float s_c4_div_c3 = s_c4/s_c3;
|
40 |
-
float s_c4_div_c3_x2 = (s_c4/s_c3)*2.0;
|
41 |
-
|
42 |
-
float s_scale_dst2 = s_c3 * s_c_scale_inv;
|
43 |
-
float s_scale_dst4 = s_c5 * s_c_scale_inv;
|
44 |
-
|
45 |
-
void OptRotateBand0(float x[1], mat3 R, out float dst[1])
|
46 |
-
{
|
47 |
-
dst[0] = x[0];
|
48 |
-
}
|
49 |
-
|
50 |
-
// 9 multiplies
|
51 |
-
void OptRotateBand1(float x[3], mat3 R, out float dst[3])
|
52 |
-
{
|
53 |
-
// derived from SlowRotateBand1
|
54 |
-
dst[0] = ( R[1][1])*x[0] + (-R[1][2])*x[1] + ( R[1][0])*x[2];
|
55 |
-
dst[1] = (-R[2][1])*x[0] + ( R[2][2])*x[1] + (-R[2][0])*x[2];
|
56 |
-
dst[2] = ( R[0][1])*x[0] + (-R[0][2])*x[1] + ( R[0][0])*x[2];
|
57 |
-
}
|
58 |
-
|
59 |
-
// 48 multiplies
|
60 |
-
void OptRotateBand2(float x[5], mat3 R, out float dst[5])
|
61 |
-
{
|
62 |
-
// Sparse matrix multiply
|
63 |
-
float sh0 = x[3] + x[4] + x[4] - x[1];
|
64 |
-
float sh1 = x[0] + s_rc2*x[2] + x[3] + x[4];
|
65 |
-
float sh2 = x[0];
|
66 |
-
float sh3 = -x[3];
|
67 |
-
float sh4 = -x[1];
|
68 |
-
|
69 |
-
// Rotations. R0 and R1 just use the raw matrix columns
|
70 |
-
float r2x = R[0][0] + R[0][1];
|
71 |
-
float r2y = R[1][0] + R[1][1];
|
72 |
-
float r2z = R[2][0] + R[2][1];
|
73 |
-
|
74 |
-
float r3x = R[0][0] + R[0][2];
|
75 |
-
float r3y = R[1][0] + R[1][2];
|
76 |
-
float r3z = R[2][0] + R[2][2];
|
77 |
-
|
78 |
-
float r4x = R[0][1] + R[0][2];
|
79 |
-
float r4y = R[1][1] + R[1][2];
|
80 |
-
float r4z = R[2][1] + R[2][2];
|
81 |
-
|
82 |
-
// dense matrix multiplication one column at a time
|
83 |
-
|
84 |
-
// column 0
|
85 |
-
float sh0_x = sh0 * R[0][0];
|
86 |
-
float sh0_y = sh0 * R[1][0];
|
87 |
-
float d0 = sh0_x * R[1][0];
|
88 |
-
float d1 = sh0_y * R[2][0];
|
89 |
-
float d2 = sh0 * (R[2][0] * R[2][0] + s_c4_div_c3);
|
90 |
-
float d3 = sh0_x * R[2][0];
|
91 |
-
float d4 = sh0_x * R[0][0] - sh0_y * R[1][0];
|
92 |
-
|
93 |
-
// column 1
|
94 |
-
float sh1_x = sh1 * R[0][2];
|
95 |
-
float sh1_y = sh1 * R[1][2];
|
96 |
-
d0 += sh1_x * R[1][2];
|
97 |
-
d1 += sh1_y * R[2][2];
|
98 |
-
d2 += sh1 * (R[2][2] * R[2][2] + s_c4_div_c3);
|
99 |
-
d3 += sh1_x * R[2][2];
|
100 |
-
d4 += sh1_x * R[0][2] - sh1_y * R[1][2];
|
101 |
-
|
102 |
-
// column 2
|
103 |
-
float sh2_x = sh2 * r2x;
|
104 |
-
float sh2_y = sh2 * r2y;
|
105 |
-
d0 += sh2_x * r2y;
|
106 |
-
d1 += sh2_y * r2z;
|
107 |
-
d2 += sh2 * (r2z * r2z + s_c4_div_c3_x2);
|
108 |
-
d3 += sh2_x * r2z;
|
109 |
-
d4 += sh2_x * r2x - sh2_y * r2y;
|
110 |
-
|
111 |
-
// column 3
|
112 |
-
float sh3_x = sh3 * r3x;
|
113 |
-
float sh3_y = sh3 * r3y;
|
114 |
-
d0 += sh3_x * r3y;
|
115 |
-
d1 += sh3_y * r3z;
|
116 |
-
d2 += sh3 * (r3z * r3z + s_c4_div_c3_x2);
|
117 |
-
d3 += sh3_x * r3z;
|
118 |
-
d4 += sh3_x * r3x - sh3_y * r3y;
|
119 |
-
|
120 |
-
// column 4
|
121 |
-
float sh4_x = sh4 * r4x;
|
122 |
-
float sh4_y = sh4 * r4y;
|
123 |
-
d0 += sh4_x * r4y;
|
124 |
-
d1 += sh4_y * r4z;
|
125 |
-
d2 += sh4 * (r4z * r4z + s_c4_div_c3_x2);
|
126 |
-
d3 += sh4_x * r4z;
|
127 |
-
d4 += sh4_x * r4x - sh4_y * r4y;
|
128 |
-
|
129 |
-
// extra multipliers
|
130 |
-
dst[0] = d0;
|
131 |
-
dst[1] = -d1;
|
132 |
-
dst[2] = d2 * s_scale_dst2;
|
133 |
-
dst[3] = -d3;
|
134 |
-
dst[4] = d4 * s_scale_dst4;
|
135 |
-
}
|
136 |
-
|
137 |
-
void main()
|
138 |
-
{
|
139 |
-
// normalization
|
140 |
-
mat3 R = mat3(ModelMat) * RotMat;
|
141 |
-
VertexOut.ModelNormal = a_Normal;
|
142 |
-
VertexOut.CameraNormal = (R * a_Normal);
|
143 |
-
VertexOut.Position = a_Position;
|
144 |
-
VertexOut.Texcoord = a_TextureCoord;
|
145 |
-
VertexOut.Tangent = (R * a_Tangent);
|
146 |
-
VertexOut.Bitangent = (R * a_Bitangent);
|
147 |
-
float PRT0, PRT1[3], PRT2[5];
|
148 |
-
PRT0 = a_PRT1[0];
|
149 |
-
PRT1[0] = a_PRT1[1];
|
150 |
-
PRT1[1] = a_PRT1[2];
|
151 |
-
PRT1[2] = a_PRT2[0];
|
152 |
-
PRT2[0] = a_PRT2[1];
|
153 |
-
PRT2[1] = a_PRT2[2];
|
154 |
-
PRT2[2] = a_PRT3[0];
|
155 |
-
PRT2[3] = a_PRT3[1];
|
156 |
-
PRT2[4] = a_PRT3[2];
|
157 |
-
|
158 |
-
OptRotateBand1(PRT1, R, PRT1);
|
159 |
-
OptRotateBand2(PRT2, R, PRT2);
|
160 |
-
|
161 |
-
VertexOut.PRT1 = vec3(PRT0,PRT1[0],PRT1[1]);
|
162 |
-
VertexOut.PRT2 = vec3(PRT1[2],PRT2[0],PRT2[1]);
|
163 |
-
VertexOut.PRT3 = vec3(PRT2[2],PRT2[3],PRT2[4]);
|
164 |
-
|
165 |
-
gl_Position = vec4(a_TextureCoord, 0.0, 1.0) - vec4(0.5, 0.5, 0, 0);
|
166 |
-
gl_Position[0] *= 2.0;
|
167 |
-
gl_Position[1] *= 2.0;
|
168 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / data /quad.fs
DELETED
@@ -1,11 +0,0 @@
|
|
1 |
-
#version 330 core
|
2 |
-
out vec4 FragColor;
|
3 |
-
|
4 |
-
in vec2 TexCoord;
|
5 |
-
|
6 |
-
uniform sampler2D screenTexture;
|
7 |
-
|
8 |
-
void main()
|
9 |
-
{
|
10 |
-
FragColor = texture(screenTexture, TexCoord);
|
11 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / data /quad.vs
DELETED
@@ -1,11 +0,0 @@
|
|
1 |
-
#version 330 core
|
2 |
-
layout (location = 0) in vec2 aPos;
|
3 |
-
layout (location = 1) in vec2 aTexCoord;
|
4 |
-
|
5 |
-
out vec2 TexCoord;
|
6 |
-
|
7 |
-
void main()
|
8 |
-
{
|
9 |
-
gl_Position = vec4(aPos.x, aPos.y, 0.0, 1.0);
|
10 |
-
TexCoord = aTexCoord;
|
11 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / framework.py
DELETED
@@ -1,95 +0,0 @@
|
|
1 |
-
# Mario Rosasco, 2016
|
2 |
-
# adapted from framework.cpp, Copyright (C) 2010-2012 by Jason L. McKesson
|
3 |
-
# This file is licensed under the MIT License.
|
4 |
-
#
|
5 |
-
# NB: Unlike in the framework.cpp organization, the main loop is contained
|
6 |
-
# in the tutorial files, not in this framework file. Additionally, a copy of
|
7 |
-
# this module file must exist in the same directory as the tutorial files
|
8 |
-
# to be imported properly.
|
9 |
-
|
10 |
-
import os
|
11 |
-
from OpenGL.GL import *
|
12 |
-
|
13 |
-
|
14 |
-
# Function that creates and compiles shaders according to the given type (a GL enum value) and
|
15 |
-
# shader program (a file containing a GLSL program).
|
16 |
-
def loadShader(shaderType, shaderFile):
|
17 |
-
# check if file exists, get full path name
|
18 |
-
strFilename = findFileOrThrow(shaderFile)
|
19 |
-
shaderData = None
|
20 |
-
with open(strFilename, 'r') as f:
|
21 |
-
shaderData = f.read()
|
22 |
-
|
23 |
-
shader = glCreateShader(shaderType)
|
24 |
-
glShaderSource(
|
25 |
-
shader,
|
26 |
-
shaderData) # note that this is a simpler function call than in C
|
27 |
-
|
28 |
-
# This shader compilation is more explicit than the one used in
|
29 |
-
# framework.cpp, which relies on a glutil wrapper function.
|
30 |
-
# This is made explicit here mainly to decrease dependence on pyOpenGL
|
31 |
-
# utilities and wrappers, which docs caution may change in future versions.
|
32 |
-
glCompileShader(shader)
|
33 |
-
|
34 |
-
status = glGetShaderiv(shader, GL_COMPILE_STATUS)
|
35 |
-
if status == GL_FALSE:
|
36 |
-
# Note that getting the error log is much simpler in Python than in C/C++
|
37 |
-
# and does not require explicit handling of the string buffer
|
38 |
-
strInfoLog = glGetShaderInfoLog(shader)
|
39 |
-
strShaderType = ""
|
40 |
-
if shaderType is GL_VERTEX_SHADER:
|
41 |
-
strShaderType = "vertex"
|
42 |
-
elif shaderType is GL_GEOMETRY_SHADER:
|
43 |
-
strShaderType = "geometry"
|
44 |
-
elif shaderType is GL_FRAGMENT_SHADER:
|
45 |
-
strShaderType = "fragment"
|
46 |
-
|
47 |
-
print("Compilation failure for " + strShaderType + " shader:\n" +
|
48 |
-
str(strInfoLog))
|
49 |
-
|
50 |
-
return shader
|
51 |
-
|
52 |
-
|
53 |
-
# Function that accepts a list of shaders, compiles them, and returns a handle to the compiled program
|
54 |
-
def createProgram(shaderList):
|
55 |
-
program = glCreateProgram()
|
56 |
-
|
57 |
-
for shader in shaderList:
|
58 |
-
glAttachShader(program, shader)
|
59 |
-
|
60 |
-
glLinkProgram(program)
|
61 |
-
|
62 |
-
status = glGetProgramiv(program, GL_LINK_STATUS)
|
63 |
-
if status == GL_FALSE:
|
64 |
-
# Note that getting the error log is much simpler in Python than in C/C++
|
65 |
-
# and does not require explicit handling of the string buffer
|
66 |
-
strInfoLog = glGetProgramInfoLog(program)
|
67 |
-
print("Linker failure: \n" + str(strInfoLog))
|
68 |
-
|
69 |
-
for shader in shaderList:
|
70 |
-
glDetachShader(program, shader)
|
71 |
-
|
72 |
-
return program
|
73 |
-
|
74 |
-
|
75 |
-
# Helper function to locate and open the target file (passed in as a string).
|
76 |
-
# Returns the full path to the file as a string.
|
77 |
-
def findFileOrThrow(strBasename):
|
78 |
-
# Keep constant names in C-style convention, for readability
|
79 |
-
# when comparing to C(/C++) code.
|
80 |
-
if os.path.isfile(strBasename):
|
81 |
-
return strBasename
|
82 |
-
|
83 |
-
LOCAL_FILE_DIR = "data" + os.sep
|
84 |
-
GLOBAL_FILE_DIR = os.path.dirname(
|
85 |
-
os.path.abspath(__file__)) + os.sep + "data" + os.sep
|
86 |
-
|
87 |
-
strFilename = LOCAL_FILE_DIR + strBasename
|
88 |
-
if os.path.isfile(strFilename):
|
89 |
-
return strFilename
|
90 |
-
|
91 |
-
strFilename = GLOBAL_FILE_DIR + strBasename
|
92 |
-
if os.path.isfile(strFilename):
|
93 |
-
return strFilename
|
94 |
-
|
95 |
-
raise IOError('Could not find target file ' + strBasename)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / glcontext.py
DELETED
@@ -1,136 +0,0 @@
|
|
1 |
-
"""Headless GPU-accelerated OpenGL context creation on Google Colaboratory.
|
2 |
-
|
3 |
-
Typical usage:
|
4 |
-
|
5 |
-
# Optional PyOpenGL configuratiopn can be done here.
|
6 |
-
# import OpenGL
|
7 |
-
# OpenGL.ERROR_CHECKING = True
|
8 |
-
|
9 |
-
# 'glcontext' must be imported before any OpenGL.* API.
|
10 |
-
from lucid.misc.gl.glcontext import create_opengl_context
|
11 |
-
|
12 |
-
# Now it's safe to import OpenGL and EGL functions
|
13 |
-
import OpenGL.GL as gl
|
14 |
-
|
15 |
-
# create_opengl_context() creates a GL context that is attached to an
|
16 |
-
# offscreen surface of the specified size. Note that rendering to buffers
|
17 |
-
# of other sizes and formats is still possible with OpenGL Framebuffers.
|
18 |
-
#
|
19 |
-
# Users are expected to directly use the EGL API in case more advanced
|
20 |
-
# context management is required.
|
21 |
-
width, height = 640, 480
|
22 |
-
create_opengl_context((width, height))
|
23 |
-
|
24 |
-
# OpenGL context is available here.
|
25 |
-
|
26 |
-
"""
|
27 |
-
|
28 |
-
from __future__ import print_function
|
29 |
-
|
30 |
-
# pylint: disable=unused-import,g-import-not-at-top,g-statement-before-imports
|
31 |
-
|
32 |
-
try:
|
33 |
-
import OpenGL
|
34 |
-
except:
|
35 |
-
print('This module depends on PyOpenGL.')
|
36 |
-
print('Please run "\033[1m!pip install -q pyopengl\033[0m" '
|
37 |
-
'prior importing this module.')
|
38 |
-
raise
|
39 |
-
|
40 |
-
import ctypes
|
41 |
-
from ctypes import pointer, util
|
42 |
-
import os
|
43 |
-
|
44 |
-
os.environ['PYOPENGL_PLATFORM'] = 'egl'
|
45 |
-
|
46 |
-
# OpenGL loading workaround.
|
47 |
-
#
|
48 |
-
# * PyOpenGL tries to load libGL, but we need libOpenGL, see [1,2].
|
49 |
-
# This could have been solved by a symlink libGL->libOpenGL, but:
|
50 |
-
#
|
51 |
-
# * Python 2.7 can't find libGL and linEGL due to a bug (see [3])
|
52 |
-
# in ctypes.util, that was only wixed in Python 3.6.
|
53 |
-
#
|
54 |
-
# So, the only solution I've found is to monkeypatch ctypes.util
|
55 |
-
# [1] https://devblogs.nvidia.com/egl-eye-opengl-visualization-without-x-server/
|
56 |
-
# [2] https://devblogs.nvidia.com/linking-opengl-server-side-rendering/
|
57 |
-
# [3] https://bugs.python.org/issue9998
|
58 |
-
_find_library_old = ctypes.util.find_library
|
59 |
-
try:
|
60 |
-
|
61 |
-
def _find_library_new(name):
|
62 |
-
return {
|
63 |
-
'GL': 'libOpenGL.so',
|
64 |
-
'EGL': 'libEGL.so',
|
65 |
-
}.get(name, _find_library_old(name))
|
66 |
-
|
67 |
-
util.find_library = _find_library_new
|
68 |
-
import OpenGL.GL as gl
|
69 |
-
import OpenGL.EGL as egl
|
70 |
-
except:
|
71 |
-
print('Unable to load OpenGL libraries. '
|
72 |
-
'Make sure you use GPU-enabled backend.')
|
73 |
-
print('Press "Runtime->Change runtime type" and set '
|
74 |
-
'"Hardware accelerator" to GPU.')
|
75 |
-
raise
|
76 |
-
finally:
|
77 |
-
util.find_library = _find_library_old
|
78 |
-
|
79 |
-
|
80 |
-
def create_opengl_context(surface_size=(640, 480)):
|
81 |
-
"""Create offscreen OpenGL context and make it current.
|
82 |
-
|
83 |
-
Users are expected to directly use EGL API in case more advanced
|
84 |
-
context management is required.
|
85 |
-
|
86 |
-
Args:
|
87 |
-
surface_size: (width, height), size of the offscreen rendering surface.
|
88 |
-
"""
|
89 |
-
egl_display = egl.eglGetDisplay(egl.EGL_DEFAULT_DISPLAY)
|
90 |
-
|
91 |
-
major, minor = egl.EGLint(), egl.EGLint()
|
92 |
-
egl.eglInitialize(egl_display, pointer(major), pointer(minor))
|
93 |
-
|
94 |
-
config_attribs = [
|
95 |
-
egl.EGL_SURFACE_TYPE, egl.EGL_PBUFFER_BIT, egl.EGL_BLUE_SIZE, 8,
|
96 |
-
egl.EGL_GREEN_SIZE, 8, egl.EGL_RED_SIZE, 8, egl.EGL_DEPTH_SIZE, 24,
|
97 |
-
egl.EGL_RENDERABLE_TYPE, egl.EGL_OPENGL_BIT, egl.EGL_NONE
|
98 |
-
]
|
99 |
-
config_attribs = (egl.EGLint * len(config_attribs))(*config_attribs)
|
100 |
-
|
101 |
-
num_configs = egl.EGLint()
|
102 |
-
egl_cfg = egl.EGLConfig()
|
103 |
-
egl.eglChooseConfig(egl_display, config_attribs, pointer(egl_cfg), 1,
|
104 |
-
pointer(num_configs))
|
105 |
-
|
106 |
-
width, height = surface_size
|
107 |
-
pbuffer_attribs = [
|
108 |
-
egl.EGL_WIDTH,
|
109 |
-
width,
|
110 |
-
egl.EGL_HEIGHT,
|
111 |
-
height,
|
112 |
-
egl.EGL_NONE,
|
113 |
-
]
|
114 |
-
pbuffer_attribs = (egl.EGLint * len(pbuffer_attribs))(*pbuffer_attribs)
|
115 |
-
egl_surf = egl.eglCreatePbufferSurface(egl_display, egl_cfg,
|
116 |
-
pbuffer_attribs)
|
117 |
-
|
118 |
-
egl.eglBindAPI(egl.EGL_OPENGL_API)
|
119 |
-
|
120 |
-
context_attribs = None
|
121 |
-
# context_attribs = [
|
122 |
-
# egl.EGL_CONTEXT_MAJOR_VERSION,
|
123 |
-
# 4,
|
124 |
-
# egl.EGL_CONTEXT_MINOR_VERSION,
|
125 |
-
# 1,
|
126 |
-
# egl.EGL_NONE,
|
127 |
-
# ]
|
128 |
-
|
129 |
-
egl_context = egl.eglCreateContext(egl_display, egl_cfg,
|
130 |
-
egl.EGL_NO_CONTEXT, context_attribs)
|
131 |
-
egl.eglMakeCurrent(egl_display, egl_surf, egl_surf, egl_context)
|
132 |
-
|
133 |
-
buffer_type = egl.EGLint()
|
134 |
-
out = egl.eglQueryContext(egl_display, egl_context,
|
135 |
-
egl.EGL_CONTEXT_CLIENT_VERSION, buffer_type)
|
136 |
-
# print(buffer_type)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / init_gl.py
DELETED
@@ -1,24 +0,0 @@
|
|
1 |
-
_glut_window = None
|
2 |
-
_context_inited = None
|
3 |
-
|
4 |
-
|
5 |
-
def initialize_GL_context(width=512, height=512, egl=False):
|
6 |
-
'''
|
7 |
-
default context uses GLUT
|
8 |
-
'''
|
9 |
-
if not egl:
|
10 |
-
import OpenGL.GLUT as GLUT
|
11 |
-
display_mode = GLUT.GLUT_DOUBLE | GLUT.GLUT_RGB | GLUT.GLUT_DEPTH
|
12 |
-
global _glut_window
|
13 |
-
if _glut_window is None:
|
14 |
-
GLUT.glutInit()
|
15 |
-
GLUT.glutInitDisplayMode(display_mode)
|
16 |
-
GLUT.glutInitWindowSize(width, height)
|
17 |
-
GLUT.glutInitWindowPosition(0, 0)
|
18 |
-
_glut_window = GLUT.glutCreateWindow("My Render.")
|
19 |
-
else:
|
20 |
-
from .glcontext import create_opengl_context
|
21 |
-
global _context_inited
|
22 |
-
if _context_inited is None:
|
23 |
-
create_opengl_context((width, height))
|
24 |
-
_context_inited = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / norm_render.py
DELETED
@@ -1,75 +0,0 @@
|
|
1 |
-
'''
|
2 |
-
MIT License
|
3 |
-
Copyright (c) 2019 Shunsuke Saito, Zeng Huang, and Ryota Natsume
|
4 |
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
5 |
-
of this software and associated documentation files (the "Software"), to deal
|
6 |
-
in the Software without restriction, including without limitation the rights
|
7 |
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
8 |
-
copies of the Software, and to permit persons to whom the Software is
|
9 |
-
furnished to do so, subject to the following conditions:
|
10 |
-
The above copyright notice and this permission notice shall be included in all
|
11 |
-
copies or substantial portions of the Software.
|
12 |
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
13 |
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
14 |
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
15 |
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
16 |
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
17 |
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
18 |
-
SOFTWARE.
|
19 |
-
'''
|
20 |
-
from OpenGL.GLUT import *
|
21 |
-
|
22 |
-
from .render2 import Render
|
23 |
-
|
24 |
-
|
25 |
-
class NormRender(Render):
|
26 |
-
def __init__(self,
|
27 |
-
width=1600,
|
28 |
-
height=1200,
|
29 |
-
name='Cam Renderer',
|
30 |
-
program_files=['simple.fs', 'simple.vs'],
|
31 |
-
color_size=1,
|
32 |
-
ms_rate=1):
|
33 |
-
Render.__init__(self, width, height, name, program_files, color_size,
|
34 |
-
ms_rate)
|
35 |
-
self.camera = None
|
36 |
-
|
37 |
-
glutDisplayFunc(self.display)
|
38 |
-
glutKeyboardFunc(self.keyboard)
|
39 |
-
|
40 |
-
def set_camera(self, camera):
|
41 |
-
self.camera = camera
|
42 |
-
self.projection_matrix, self.model_view_matrix = camera.get_gl_matrix()
|
43 |
-
|
44 |
-
def set_matrices(self, projection, modelview):
|
45 |
-
self.projection_matrix = projection
|
46 |
-
self.model_view_matrix = modelview
|
47 |
-
|
48 |
-
def keyboard(self, key, x, y):
|
49 |
-
# up
|
50 |
-
eps = 1
|
51 |
-
# print(key)
|
52 |
-
if key == b'w':
|
53 |
-
self.camera.center += eps * self.camera.direction
|
54 |
-
elif key == b's':
|
55 |
-
self.camera.center -= eps * self.camera.direction
|
56 |
-
if key == b'a':
|
57 |
-
self.camera.center -= eps * self.camera.right
|
58 |
-
elif key == b'd':
|
59 |
-
self.camera.center += eps * self.camera.right
|
60 |
-
if key == b' ':
|
61 |
-
self.camera.center += eps * self.camera.up
|
62 |
-
elif key == b'x':
|
63 |
-
self.camera.center -= eps * self.camera.up
|
64 |
-
elif key == b'i':
|
65 |
-
self.camera.near += 0.1 * eps
|
66 |
-
self.camera.far += 0.1 * eps
|
67 |
-
elif key == b'o':
|
68 |
-
self.camera.near -= 0.1 * eps
|
69 |
-
self.camera.far -= 0.1 * eps
|
70 |
-
|
71 |
-
self.projection_matrix, self.model_view_matrix = self.camera.get_gl_matrix(
|
72 |
-
)
|
73 |
-
|
74 |
-
def show(self):
|
75 |
-
glutMainLoop()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl / render.py
DELETED
@@ -1,380 +0,0 @@
|
|
1 |
-
|
2 |
-
# -*- coding: utf-8 -*-
|
3 |
-
|
4 |
-
# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
|
5 |
-
# holder of all proprietary rights on this computer program.
|
6 |
-
# You can only use this computer program if you have closed
|
7 |
-
# a license agreement with MPG or you get the right to use the computer
|
8 |
-
# program from someone who is authorized to grant you that right.
|
9 |
-
# Any use of the computer program without a valid license is prohibited and
|
10 |
-
# liable to prosecution.
|
11 |
-
#
|
12 |
-
# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
|
13 |
-
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
|
14 |
-
# for Intelligent Systems. All rights reserved.
|
15 |
-
#
|
16 |
-
# Contact: [email protected]
|
17 |
-
|
18 |
-
from ctypes import *
|
19 |
-
|
20 |
-
import numpy as np
|
21 |
-
from .framework import *
|
22 |
-
|
23 |
-
GLUT = None
|
24 |
-
|
25 |
-
|
26 |
-
# NOTE: Render class assumes GL context is created already.
|
27 |
-
class Render:
|
28 |
-
def __init__(self,
|
29 |
-
width=1600,
|
30 |
-
height=1200,
|
31 |
-
name='GL Renderer',
|
32 |
-
program_files=['simple.fs', 'simple.vs'],
|
33 |
-
color_size=1,
|
34 |
-
ms_rate=1,
|
35 |
-
egl=False):
|
36 |
-
self.width = width
|
37 |
-
self.height = height
|
38 |
-
self.name = name
|
39 |
-
self.use_inverse_depth = False
|
40 |
-
self.egl = egl
|
41 |
-
|
42 |
-
glEnable(GL_DEPTH_TEST)
|
43 |
-
|
44 |
-
glClampColor(GL_CLAMP_READ_COLOR, GL_FALSE)
|
45 |
-
glClampColor(GL_CLAMP_FRAGMENT_COLOR, GL_FALSE)
|
46 |
-
glClampColor(GL_CLAMP_VERTEX_COLOR, GL_FALSE)
|
47 |
-
|
48 |
-
# init program
|
49 |
-
shader_list = []
|
50 |
-
|
51 |
-
for program_file in program_files:
|
52 |
-
_, ext = os.path.splitext(program_file)
|
53 |
-
if ext == '.vs':
|
54 |
-
shader_list.append(loadShader(GL_VERTEX_SHADER, program_file))
|
55 |
-
elif ext == '.fs':
|
56 |
-
shader_list.append(loadShader(GL_FRAGMENT_SHADER,
|
57 |
-
program_file))
|
58 |
-
elif ext == '.gs':
|
59 |
-
shader_list.append(loadShader(GL_GEOMETRY_SHADER,
|
60 |
-
program_file))
|
61 |
-
|
62 |
-
self.program = createProgram(shader_list)
|
63 |
-
|
64 |
-
for shader in shader_list:
|
65 |
-
glDeleteShader(shader)
|
66 |
-
|
67 |
-
# Init uniform variables
|
68 |
-
self.model_mat_unif = glGetUniformLocation(self.program, 'ModelMat')
|
69 |
-
self.persp_mat_unif = glGetUniformLocation(self.program, 'PerspMat')
|
70 |
-
|
71 |
-
self.vertex_buffer = glGenBuffers(1)
|
72 |
-
|
73 |
-
# Init screen quad program and buffer
|
74 |
-
self.quad_program, self.quad_buffer = self.init_quad_program()
|
75 |
-
|
76 |
-
# Configure frame buffer
|
77 |
-
self.frame_buffer = glGenFramebuffers(1)
|
78 |
-
glBindFramebuffer(GL_FRAMEBUFFER, self.frame_buffer)
|
79 |
-
|
80 |
-
self.intermediate_fbo = None
|
81 |
-
if ms_rate > 1:
|
82 |
-
# Configure texture buffer to render to
|
83 |
-
self.color_buffer = []
|
84 |
-
for i in range(color_size):
|
85 |
-
color_buffer = glGenTextures(1)
|
86 |
-
multi_sample_rate = ms_rate
|
87 |
-
glBindTexture(GL_TEXTURE_2D_MULTISAMPLE, color_buffer)
|
88 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,
|
89 |
-
GL_CLAMP_TO_EDGE)
|
90 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,
|
91 |
-
GL_CLAMP_TO_EDGE)
|
92 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
|
93 |
-
GL_LINEAR)
|
94 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
|
95 |
-
GL_LINEAR)
|
96 |
-
glTexImage2DMultisample(GL_TEXTURE_2D_MULTISAMPLE,
|
97 |
-
multi_sample_rate, GL_RGBA32F,
|
98 |
-
self.width, self.height, GL_TRUE)
|
99 |
-
glBindTexture(GL_TEXTURE_2D_MULTISAMPLE, 0)
|
100 |
-
glFramebufferTexture2D(GL_FRAMEBUFFER,
|
101 |
-
GL_COLOR_ATTACHMENT0 + i,
|
102 |
-
GL_TEXTURE_2D_MULTISAMPLE, color_buffer,
|
103 |
-
0)
|
104 |
-
self.color_buffer.append(color_buffer)
|
105 |
-
|
106 |
-
self.render_buffer = glGenRenderbuffers(1)
|
107 |
-
glBindRenderbuffer(GL_RENDERBUFFER, self.render_buffer)
|
108 |
-
glRenderbufferStorageMultisample(GL_RENDERBUFFER,
|
109 |
-
multi_sample_rate,
|
110 |
-
GL_DEPTH24_STENCIL8, self.width,
|
111 |
-
self.height)
|
112 |
-
glBindRenderbuffer(GL_RENDERBUFFER, 0)
|
113 |
-
glFramebufferRenderbuffer(GL_FRAMEBUFFER,
|
114 |
-
GL_DEPTH_STENCIL_ATTACHMENT,
|
115 |
-
GL_RENDERBUFFER, self.render_buffer)
|
116 |
-
|
117 |
-
attachments = []
|
118 |
-
for i in range(color_size):
|
119 |
-
attachments.append(GL_COLOR_ATTACHMENT0 + i)
|
120 |
-
glDrawBuffers(color_size, attachments)
|
121 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
122 |
-
|
123 |
-
self.intermediate_fbo = glGenFramebuffers(1)
|
124 |
-
glBindFramebuffer(GL_FRAMEBUFFER, self.intermediate_fbo)
|
125 |
-
|
126 |
-
self.screen_texture = []
|
127 |
-
for i in range(color_size):
|
128 |
-
screen_texture = glGenTextures(1)
|
129 |
-
glBindTexture(GL_TEXTURE_2D, screen_texture)
|
130 |
-
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, self.width,
|
131 |
-
self.height, 0, GL_RGBA, GL_FLOAT, None)
|
132 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
|
133 |
-
GL_LINEAR)
|
134 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
|
135 |
-
GL_LINEAR)
|
136 |
-
glFramebufferTexture2D(GL_FRAMEBUFFER,
|
137 |
-
GL_COLOR_ATTACHMENT0 + i, GL_TEXTURE_2D,
|
138 |
-
screen_texture, 0)
|
139 |
-
self.screen_texture.append(screen_texture)
|
140 |
-
|
141 |
-
glDrawBuffers(color_size, attachments)
|
142 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
143 |
-
else:
|
144 |
-
self.color_buffer = []
|
145 |
-
for i in range(color_size):
|
146 |
-
color_buffer = glGenTextures(1)
|
147 |
-
glBindTexture(GL_TEXTURE_2D, color_buffer)
|
148 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,
|
149 |
-
GL_CLAMP_TO_EDGE)
|
150 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,
|
151 |
-
GL_CLAMP_TO_EDGE)
|
152 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
|
153 |
-
GL_NEAREST)
|
154 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
|
155 |
-
GL_NEAREST)
|
156 |
-
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, self.width,
|
157 |
-
self.height, 0, GL_RGBA, GL_FLOAT, None)
|
158 |
-
glFramebufferTexture2D(GL_FRAMEBUFFER,
|
159 |
-
GL_COLOR_ATTACHMENT0 + i, GL_TEXTURE_2D,
|
160 |
-
color_buffer, 0)
|
161 |
-
self.color_buffer.append(color_buffer)
|
162 |
-
|
163 |
-
# Configure depth texture map to render to
|
164 |
-
self.depth_buffer = glGenTextures(1)
|
165 |
-
glBindTexture(GL_TEXTURE_2D, self.depth_buffer)
|
166 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
|
167 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)
|
168 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
|
169 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
|
170 |
-
glTexParameteri(GL_TEXTURE_2D, GL_DEPTH_TEXTURE_MODE, GL_INTENSITY)
|
171 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_MODE,
|
172 |
-
GL_COMPARE_R_TO_TEXTURE)
|
173 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FUNC, GL_LEQUAL)
|
174 |
-
glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT, self.width,
|
175 |
-
self.height, 0, GL_DEPTH_COMPONENT, GL_FLOAT, None)
|
176 |
-
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT,
|
177 |
-
GL_TEXTURE_2D, self.depth_buffer, 0)
|
178 |
-
|
179 |
-
attachments = []
|
180 |
-
for i in range(color_size):
|
181 |
-
attachments.append(GL_COLOR_ATTACHMENT0 + i)
|
182 |
-
glDrawBuffers(color_size, attachments)
|
183 |
-
self.screen_texture = self.color_buffer
|
184 |
-
|
185 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
186 |
-
|
187 |
-
# Configure texture buffer if needed
|
188 |
-
self.render_texture = None
|
189 |
-
|
190 |
-
# NOTE: original render_texture only support one input
|
191 |
-
# this is tentative member of this issue
|
192 |
-
self.render_texture_v2 = {}
|
193 |
-
|
194 |
-
# Inner storage for buffer data
|
195 |
-
self.vertex_data = None
|
196 |
-
self.vertex_dim = None
|
197 |
-
self.n_vertices = None
|
198 |
-
|
199 |
-
self.model_view_matrix = None
|
200 |
-
self.projection_matrix = None
|
201 |
-
|
202 |
-
if not egl:
|
203 |
-
global GLUT
|
204 |
-
import OpenGL.GLUT as GLUT
|
205 |
-
GLUT.glutDisplayFunc(self.display)
|
206 |
-
|
207 |
-
def init_quad_program(self):
|
208 |
-
shader_list = []
|
209 |
-
|
210 |
-
shader_list.append(loadShader(GL_VERTEX_SHADER, "quad.vs"))
|
211 |
-
shader_list.append(loadShader(GL_FRAGMENT_SHADER, "quad.fs"))
|
212 |
-
|
213 |
-
the_program = createProgram(shader_list)
|
214 |
-
|
215 |
-
for shader in shader_list:
|
216 |
-
glDeleteShader(shader)
|
217 |
-
|
218 |
-
# vertex attributes for a quad that fills the entire screen in Normalized Device Coordinates.
|
219 |
-
# positions # texCoords
|
220 |
-
quad_vertices = np.array([
|
221 |
-
-1.0, 1.0, 0.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0, -1.0, 1.0, 0.0,
|
222 |
-
-1.0, 1.0, 0.0, 1.0, 1.0, -1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0
|
223 |
-
])
|
224 |
-
|
225 |
-
quad_buffer = glGenBuffers(1)
|
226 |
-
glBindBuffer(GL_ARRAY_BUFFER, quad_buffer)
|
227 |
-
glBufferData(GL_ARRAY_BUFFER, quad_vertices, GL_STATIC_DRAW)
|
228 |
-
|
229 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
230 |
-
|
231 |
-
return the_program, quad_buffer
|
232 |
-
|
233 |
-
def set_mesh(self, vertices, faces):
|
234 |
-
self.vertex_data = vertices[faces.reshape([-1])]
|
235 |
-
self.vertex_dim = self.vertex_data.shape[1]
|
236 |
-
self.n_vertices = self.vertex_data.shape[0]
|
237 |
-
|
238 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.vertex_buffer)
|
239 |
-
glBufferData(GL_ARRAY_BUFFER, self.vertex_data, GL_STATIC_DRAW)
|
240 |
-
|
241 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
242 |
-
|
243 |
-
def set_viewpoint(self, projection, model_view):
|
244 |
-
self.projection_matrix = projection
|
245 |
-
self.model_view_matrix = model_view
|
246 |
-
|
247 |
-
def draw_init(self):
|
248 |
-
glBindFramebuffer(GL_FRAMEBUFFER, self.frame_buffer)
|
249 |
-
glEnable(GL_DEPTH_TEST)
|
250 |
-
|
251 |
-
glClearColor(0.0, 0.0, 0.0, 0.0)
|
252 |
-
if self.use_inverse_depth:
|
253 |
-
glDepthFunc(GL_GREATER)
|
254 |
-
glClearDepth(0.0)
|
255 |
-
else:
|
256 |
-
glDepthFunc(GL_LESS)
|
257 |
-
glClearDepth(1.0)
|
258 |
-
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
|
259 |
-
|
260 |
-
def draw_end(self):
|
261 |
-
if self.intermediate_fbo is not None:
|
262 |
-
for i in range(len(self.color_buffer)):
|
263 |
-
glBindFramebuffer(GL_READ_FRAMEBUFFER, self.frame_buffer)
|
264 |
-
glReadBuffer(GL_COLOR_ATTACHMENT0 + i)
|
265 |
-
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self.intermediate_fbo)
|
266 |
-
glDrawBuffer(GL_COLOR_ATTACHMENT0 + i)
|
267 |
-
glBlitFramebuffer(0, 0, self.width, self.height, 0, 0,
|
268 |
-
self.width, self.height, GL_COLOR_BUFFER_BIT,
|
269 |
-
GL_NEAREST)
|
270 |
-
|
271 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
272 |
-
glDepthFunc(GL_LESS)
|
273 |
-
glClearDepth(1.0)
|
274 |
-
|
275 |
-
def draw(self):
|
276 |
-
self.draw_init()
|
277 |
-
|
278 |
-
glUseProgram(self.program)
|
279 |
-
glUniformMatrix4fv(self.model_mat_unif, 1, GL_FALSE,
|
280 |
-
self.model_view_matrix.transpose())
|
281 |
-
glUniformMatrix4fv(self.persp_mat_unif, 1, GL_FALSE,
|
282 |
-
self.projection_matrix.transpose())
|
283 |
-
|
284 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.vertex_buffer)
|
285 |
-
|
286 |
-
glEnableVertexAttribArray(0)
|
287 |
-
glVertexAttribPointer(0, self.vertex_dim, GL_DOUBLE, GL_FALSE, 0, None)
|
288 |
-
|
289 |
-
glDrawArrays(GL_TRIANGLES, 0, self.n_vertices)
|
290 |
-
|
291 |
-
glDisableVertexAttribArray(0)
|
292 |
-
|
293 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
294 |
-
|
295 |
-
glUseProgram(0)
|
296 |
-
|
297 |
-
self.draw_end()
|
298 |
-
|
299 |
-
def get_color(self, color_id=0):
|
300 |
-
glBindFramebuffer(
|
301 |
-
GL_FRAMEBUFFER, self.intermediate_fbo
|
302 |
-
if self.intermediate_fbo is not None else self.frame_buffer)
|
303 |
-
glReadBuffer(GL_COLOR_ATTACHMENT0 + color_id)
|
304 |
-
data = glReadPixels(0,
|
305 |
-
0,
|
306 |
-
self.width,
|
307 |
-
self.height,
|
308 |
-
GL_RGBA,
|
309 |
-
GL_FLOAT,
|
310 |
-
outputType=None)
|
311 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
312 |
-
rgb = data.reshape(self.height, self.width, -1)
|
313 |
-
rgb = np.flip(rgb, 0)
|
314 |
-
return rgb
|
315 |
-
|
316 |
-
def get_z_value(self):
|
317 |
-
glBindFramebuffer(GL_FRAMEBUFFER, self.frame_buffer)
|
318 |
-
data = glReadPixels(0,
|
319 |
-
0,
|
320 |
-
self.width,
|
321 |
-
self.height,
|
322 |
-
GL_DEPTH_COMPONENT,
|
323 |
-
GL_FLOAT,
|
324 |
-
outputType=None)
|
325 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
326 |
-
z = data.reshape(self.height, self.width)
|
327 |
-
z = np.flip(z, 0)
|
328 |
-
return z
|
329 |
-
|
330 |
-
def display(self):
|
331 |
-
self.draw()
|
332 |
-
|
333 |
-
if not self.egl:
|
334 |
-
# First we draw a scene.
|
335 |
-
# Notice the result is stored in the texture buffer.
|
336 |
-
|
337 |
-
# Then we return to the default frame buffer since we will display on the screen.
|
338 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
339 |
-
|
340 |
-
# Do the clean-up.
|
341 |
-
glClearColor(0.0, 0.0, 0.0, 0.0)
|
342 |
-
glClear(GL_COLOR_BUFFER_BIT)
|
343 |
-
|
344 |
-
# We draw a rectangle which covers the whole screen.
|
345 |
-
glUseProgram(self.quad_program)
|
346 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.quad_buffer)
|
347 |
-
|
348 |
-
size_of_double = 8
|
349 |
-
glEnableVertexAttribArray(0)
|
350 |
-
glVertexAttribPointer(0, 2, GL_DOUBLE, GL_FALSE,
|
351 |
-
4 * size_of_double, None)
|
352 |
-
glEnableVertexAttribArray(1)
|
353 |
-
glVertexAttribPointer(1, 2, GL_DOUBLE, GL_FALSE,
|
354 |
-
4 * size_of_double,
|
355 |
-
c_void_p(2 * size_of_double))
|
356 |
-
|
357 |
-
glDisable(GL_DEPTH_TEST)
|
358 |
-
|
359 |
-
# The stored texture is then mapped to this rectangle.
|
360 |
-
# properly assing color buffer texture
|
361 |
-
glActiveTexture(GL_TEXTURE0)
|
362 |
-
glBindTexture(GL_TEXTURE_2D, self.screen_texture[0])
|
363 |
-
glUniform1i(
|
364 |
-
glGetUniformLocation(self.quad_program, 'screenTexture'), 0)
|
365 |
-
|
366 |
-
glDrawArrays(GL_TRIANGLES, 0, 6)
|
367 |
-
|
368 |
-
glDisableVertexAttribArray(1)
|
369 |
-
glDisableVertexAttribArray(0)
|
370 |
-
|
371 |
-
glEnable(GL_DEPTH_TEST)
|
372 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
373 |
-
glUseProgram(0)
|
374 |
-
|
375 |
-
GLUT.glutSwapBuffers()
|
376 |
-
GLUT.glutPostRedisplay()
|
377 |
-
|
378 |
-
def show(self):
|
379 |
-
if not self.egl:
|
380 |
-
GLUT.glutMainLoop()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl /cam_render.py
DELETED
@@ -1,80 +0,0 @@
|
|
1 |
-
|
2 |
-
# -*- coding: utf-8 -*-
|
3 |
-
|
4 |
-
# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
|
5 |
-
# holder of all proprietary rights on this computer program.
|
6 |
-
# You can only use this computer program if you have closed
|
7 |
-
# a license agreement with MPG or you get the right to use the computer
|
8 |
-
# program from someone who is authorized to grant you that right.
|
9 |
-
# Any use of the computer program without a valid license is prohibited and
|
10 |
-
# liable to prosecution.
|
11 |
-
#
|
12 |
-
# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
|
13 |
-
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
|
14 |
-
# for Intelligent Systems. All rights reserved.
|
15 |
-
#
|
16 |
-
# Contact: [email protected]
|
17 |
-
|
18 |
-
from .render import Render
|
19 |
-
|
20 |
-
GLUT = None
|
21 |
-
|
22 |
-
|
23 |
-
class CamRender(Render):
|
24 |
-
def __init__(self,
|
25 |
-
width=1600,
|
26 |
-
height=1200,
|
27 |
-
name='Cam Renderer',
|
28 |
-
program_files=['simple.fs', 'simple.vs'],
|
29 |
-
color_size=1,
|
30 |
-
ms_rate=1,
|
31 |
-
egl=False):
|
32 |
-
Render.__init__(self,
|
33 |
-
width,
|
34 |
-
height,
|
35 |
-
name,
|
36 |
-
program_files,
|
37 |
-
color_size,
|
38 |
-
ms_rate=ms_rate,
|
39 |
-
egl=egl)
|
40 |
-
self.camera = None
|
41 |
-
|
42 |
-
if not egl:
|
43 |
-
global GLUT
|
44 |
-
import OpenGL.GLUT as GLUT
|
45 |
-
GLUT.glutDisplayFunc(self.display)
|
46 |
-
GLUT.glutKeyboardFunc(self.keyboard)
|
47 |
-
|
48 |
-
def set_camera(self, camera):
|
49 |
-
self.camera = camera
|
50 |
-
self.projection_matrix, self.model_view_matrix = camera.get_gl_matrix()
|
51 |
-
|
52 |
-
def keyboard(self, key, x, y):
|
53 |
-
# up
|
54 |
-
eps = 1
|
55 |
-
# print(key)
|
56 |
-
if key == b'w':
|
57 |
-
self.camera.center += eps * self.camera.direction
|
58 |
-
elif key == b's':
|
59 |
-
self.camera.center -= eps * self.camera.direction
|
60 |
-
if key == b'a':
|
61 |
-
self.camera.center -= eps * self.camera.right
|
62 |
-
elif key == b'd':
|
63 |
-
self.camera.center += eps * self.camera.right
|
64 |
-
if key == b' ':
|
65 |
-
self.camera.center += eps * self.camera.up
|
66 |
-
elif key == b'x':
|
67 |
-
self.camera.center -= eps * self.camera.up
|
68 |
-
elif key == b'i':
|
69 |
-
self.camera.near += 0.1 * eps
|
70 |
-
self.camera.far += 0.1 * eps
|
71 |
-
elif key == b'o':
|
72 |
-
self.camera.near -= 0.1 * eps
|
73 |
-
self.camera.far -= 0.1 * eps
|
74 |
-
|
75 |
-
self.projection_matrix, self.model_view_matrix = self.camera.get_gl_matrix(
|
76 |
-
)
|
77 |
-
|
78 |
-
def show(self):
|
79 |
-
if GLUT is not None:
|
80 |
-
GLUT.glutMainLoop()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl /color_render.py
DELETED
@@ -1,158 +0,0 @@
|
|
1 |
-
'''
|
2 |
-
MIT License
|
3 |
-
Copyright (c) 2019 Shunsuke Saito, Zeng Huang, and Ryota Natsume
|
4 |
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
5 |
-
of this software and associated documentation files (the "Software"), to deal
|
6 |
-
in the Software without restriction, including without limitation the rights
|
7 |
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
8 |
-
copies of the Software, and to permit persons to whom the Software is
|
9 |
-
furnished to do so, subject to the following conditions:
|
10 |
-
The above copyright notice and this permission notice shall be included in all
|
11 |
-
copies or substantial portions of the Software.
|
12 |
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
13 |
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
14 |
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
15 |
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
16 |
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
17 |
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
18 |
-
SOFTWARE.
|
19 |
-
'''
|
20 |
-
import numpy as np
|
21 |
-
import random
|
22 |
-
|
23 |
-
from .framework import *
|
24 |
-
from .cam_render import CamRender
|
25 |
-
|
26 |
-
|
27 |
-
class ColorRender(CamRender):
|
28 |
-
def __init__(self, width=1600, height=1200, name='Color Renderer', egl=False):
|
29 |
-
program_files = ['color.vs', 'color.fs']
|
30 |
-
CamRender.__init__(self,
|
31 |
-
width,
|
32 |
-
height,
|
33 |
-
name,
|
34 |
-
program_files=program_files,
|
35 |
-
color_size=3, egl=egl)
|
36 |
-
|
37 |
-
# WARNING: this differs from vertex_buffer and vertex_data in Render
|
38 |
-
self.vert_buffer = {}
|
39 |
-
self.vert_data = {}
|
40 |
-
|
41 |
-
# normal
|
42 |
-
self.norm_buffer = {}
|
43 |
-
self.norm_data = {}
|
44 |
-
|
45 |
-
self.color_buffer = {}
|
46 |
-
self.color_data = {}
|
47 |
-
|
48 |
-
self.vertex_dim = {}
|
49 |
-
self.n_vertices = {}
|
50 |
-
|
51 |
-
self.rot_mat_unif = glGetUniformLocation(self.program, 'RotMat')
|
52 |
-
self.rot_matrix = np.eye(3)
|
53 |
-
|
54 |
-
self.norm_mat_unif = glGetUniformLocation(self.program, 'NormMat')
|
55 |
-
self.normalize_matrix = np.eye(4)
|
56 |
-
|
57 |
-
def set_norm_mat(self, scale, center):
|
58 |
-
N = np.eye(4)
|
59 |
-
N[:3, :3] = scale * np.eye(3)
|
60 |
-
N[:3, 3] = -scale * center
|
61 |
-
|
62 |
-
self.normalize_matrix = N
|
63 |
-
|
64 |
-
def set_mesh(self, vertices, faces, color, normals, mat_name='all'):
|
65 |
-
|
66 |
-
self.vert_data[mat_name] = vertices[faces.reshape([-1])]
|
67 |
-
self.n_vertices[mat_name] = self.vert_data[mat_name].shape[0]
|
68 |
-
self.vertex_dim[mat_name] = self.vert_data[mat_name].shape[1]
|
69 |
-
self.color_data[mat_name] = color[faces.reshape([-1])]
|
70 |
-
self.norm_data[mat_name] = normals[faces.reshape([-1])]
|
71 |
-
|
72 |
-
if mat_name not in self.vert_buffer.keys():
|
73 |
-
self.vert_buffer[mat_name] = glGenBuffers(1)
|
74 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.vert_buffer[mat_name])
|
75 |
-
glBufferData(GL_ARRAY_BUFFER, self.vert_data[mat_name], GL_STATIC_DRAW)
|
76 |
-
|
77 |
-
if mat_name not in self.color_buffer.keys():
|
78 |
-
self.color_buffer[mat_name] = glGenBuffers(1)
|
79 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.color_buffer[mat_name])
|
80 |
-
glBufferData(GL_ARRAY_BUFFER, self.color_data[mat_name],
|
81 |
-
GL_STATIC_DRAW)
|
82 |
-
|
83 |
-
if mat_name not in self.norm_buffer.keys():
|
84 |
-
self.norm_buffer[mat_name] = glGenBuffers(1)
|
85 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer[mat_name])
|
86 |
-
glBufferData(GL_ARRAY_BUFFER, self.norm_data[mat_name], GL_STATIC_DRAW)
|
87 |
-
|
88 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
89 |
-
|
90 |
-
def cleanup(self):
|
91 |
-
|
92 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
93 |
-
|
94 |
-
for key in self.vert_data:
|
95 |
-
glDeleteBuffers(1, [self.vert_buffer[key]])
|
96 |
-
glDeleteBuffers(1, [self.color_buffer[key]])
|
97 |
-
glDeleteBuffers(1, [self.norm_buffer[key]])
|
98 |
-
|
99 |
-
self.norm_buffer = {}
|
100 |
-
self.norm_data = {}
|
101 |
-
|
102 |
-
self.vert_buffer = {}
|
103 |
-
self.vert_data = {}
|
104 |
-
|
105 |
-
self.color_buffer = {}
|
106 |
-
self.color_data = {}
|
107 |
-
|
108 |
-
self.render_texture_mat = {}
|
109 |
-
|
110 |
-
self.vertex_dim = {}
|
111 |
-
self.n_vertices = {}
|
112 |
-
|
113 |
-
def draw(self):
|
114 |
-
self.draw_init()
|
115 |
-
|
116 |
-
glEnable(GL_MULTISAMPLE)
|
117 |
-
|
118 |
-
glUseProgram(self.program)
|
119 |
-
glUniformMatrix4fv(self.norm_mat_unif, 1, GL_FALSE,
|
120 |
-
self.normalize_matrix.transpose())
|
121 |
-
glUniformMatrix4fv(self.model_mat_unif, 1, GL_FALSE,
|
122 |
-
self.model_view_matrix.transpose())
|
123 |
-
glUniformMatrix4fv(self.persp_mat_unif, 1, GL_FALSE,
|
124 |
-
self.projection_matrix.transpose())
|
125 |
-
glUniformMatrix3fv(self.rot_mat_unif, 1, GL_FALSE,
|
126 |
-
self.rot_matrix.transpose())
|
127 |
-
|
128 |
-
for mat in self.vert_buffer:
|
129 |
-
|
130 |
-
# Handle vertex buffer
|
131 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.vert_buffer[mat])
|
132 |
-
glEnableVertexAttribArray(0)
|
133 |
-
glVertexAttribPointer(0, self.vertex_dim[mat], GL_DOUBLE, GL_FALSE,
|
134 |
-
0, None)
|
135 |
-
|
136 |
-
# Handle color buffer
|
137 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.color_buffer[mat])
|
138 |
-
glEnableVertexAttribArray(1)
|
139 |
-
glVertexAttribPointer(1, 3, GL_DOUBLE, GL_FALSE, 0, None)
|
140 |
-
|
141 |
-
# Handle normal buffer
|
142 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer[mat])
|
143 |
-
glEnableVertexAttribArray(2)
|
144 |
-
glVertexAttribPointer(2, 3, GL_DOUBLE, GL_FALSE, 0, None)
|
145 |
-
|
146 |
-
glDrawArrays(GL_TRIANGLES, 0, self.n_vertices[mat])
|
147 |
-
|
148 |
-
glDisableVertexAttribArray(2)
|
149 |
-
glDisableVertexAttribArray(1)
|
150 |
-
glDisableVertexAttribArray(0)
|
151 |
-
|
152 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
153 |
-
|
154 |
-
glUseProgram(0)
|
155 |
-
|
156 |
-
glDisable(GL_MULTISAMPLE)
|
157 |
-
|
158 |
-
self.draw_end()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl /normal_render.py
DELETED
@@ -1,93 +0,0 @@
|
|
1 |
-
'''
|
2 |
-
MIT License
|
3 |
-
Copyright (c) 2019 Shunsuke Saito, Zeng Huang, and Ryota Natsume
|
4 |
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
5 |
-
of this software and associated documentation files (the "Software"), to deal
|
6 |
-
in the Software without restriction, including without limitation the rights
|
7 |
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
8 |
-
copies of the Software, and to permit persons to whom the Software is
|
9 |
-
furnished to do so, subject to the following conditions:
|
10 |
-
The above copyright notice and this permission notice shall be included in all
|
11 |
-
copies or substantial portions of the Software.
|
12 |
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
13 |
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
14 |
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
15 |
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
16 |
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
17 |
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
18 |
-
SOFTWARE.
|
19 |
-
'''
|
20 |
-
import numpy as np
|
21 |
-
import math
|
22 |
-
|
23 |
-
from .framework import *
|
24 |
-
from .norm_render import NormRender
|
25 |
-
|
26 |
-
|
27 |
-
class NormalRender(NormRender):
|
28 |
-
def __init__(self, width=1600, height=1200, name='Normal Renderer'):
|
29 |
-
NormRender.__init__(self,
|
30 |
-
width,
|
31 |
-
height,
|
32 |
-
name,
|
33 |
-
program_files=['normal.vs', 'normal.fs'])
|
34 |
-
|
35 |
-
self.norm_buffer = glGenBuffers(1)
|
36 |
-
|
37 |
-
self.norm_data = None
|
38 |
-
|
39 |
-
def set_normal_mesh(self, vertices, faces, norms, face_normals):
|
40 |
-
NormRender.set_mesh(self, vertices, faces)
|
41 |
-
|
42 |
-
self.norm_data = norms[face_normals.reshape([-1])]
|
43 |
-
|
44 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer)
|
45 |
-
glBufferData(GL_ARRAY_BUFFER, self.norm_data, GL_STATIC_DRAW)
|
46 |
-
|
47 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
48 |
-
|
49 |
-
def euler_to_rot_mat(self, r_x, r_y, r_z):
|
50 |
-
R_x = np.array([[1, 0, 0], [0, math.cos(r_x), -math.sin(r_x)],
|
51 |
-
[0, math.sin(r_x), math.cos(r_x)]])
|
52 |
-
|
53 |
-
R_y = np.array([[math.cos(r_y), 0, math.sin(r_y)], [0, 1, 0],
|
54 |
-
[-math.sin(r_y), 0, math.cos(r_y)]])
|
55 |
-
|
56 |
-
R_z = np.array([[math.cos(r_z), -math.sin(r_z), 0],
|
57 |
-
[math.sin(r_z), math.cos(r_z), 0], [0, 0, 1]])
|
58 |
-
|
59 |
-
R = np.dot(R_z, np.dot(R_y, R_x))
|
60 |
-
|
61 |
-
return R
|
62 |
-
|
63 |
-
def draw(self):
|
64 |
-
self.draw_init()
|
65 |
-
|
66 |
-
glUseProgram(self.program)
|
67 |
-
glUniformMatrix4fv(self.model_mat_unif, 1, GL_FALSE,
|
68 |
-
self.model_view_matrix.transpose())
|
69 |
-
glUniformMatrix4fv(self.persp_mat_unif, 1, GL_FALSE,
|
70 |
-
self.projection_matrix.transpose())
|
71 |
-
|
72 |
-
# Handle vertex buffer
|
73 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.vertex_buffer)
|
74 |
-
|
75 |
-
glEnableVertexAttribArray(0)
|
76 |
-
glVertexAttribPointer(0, self.vertex_dim, GL_DOUBLE, GL_FALSE, 0, None)
|
77 |
-
|
78 |
-
# Handle normal buffer
|
79 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer)
|
80 |
-
|
81 |
-
glEnableVertexAttribArray(1)
|
82 |
-
glVertexAttribPointer(1, 3, GL_DOUBLE, GL_FALSE, 0, None)
|
83 |
-
|
84 |
-
glDrawArrays(GL_TRIANGLES, 0, self.n_vertices)
|
85 |
-
|
86 |
-
glDisableVertexAttribArray(1)
|
87 |
-
glDisableVertexAttribArray(0)
|
88 |
-
|
89 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
90 |
-
|
91 |
-
glUseProgram(0)
|
92 |
-
|
93 |
-
self.draw_end()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl /prt_render.py
DELETED
@@ -1,450 +0,0 @@
|
|
1 |
-
|
2 |
-
# -*- coding: utf-8 -*-
|
3 |
-
|
4 |
-
# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
|
5 |
-
# holder of all proprietary rights on this computer program.
|
6 |
-
# You can only use this computer program if you have closed
|
7 |
-
# a license agreement with MPG or you get the right to use the computer
|
8 |
-
# program from someone who is authorized to grant you that right.
|
9 |
-
# Any use of the computer program without a valid license is prohibited and
|
10 |
-
# liable to prosecution.
|
11 |
-
#
|
12 |
-
# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
|
13 |
-
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
|
14 |
-
# for Intelligent Systems. All rights reserved.
|
15 |
-
#
|
16 |
-
# Contact: [email protected]
|
17 |
-
|
18 |
-
import numpy as np
|
19 |
-
import random
|
20 |
-
|
21 |
-
from .framework import *
|
22 |
-
from .cam_render import CamRender
|
23 |
-
|
24 |
-
|
25 |
-
class PRTRender(CamRender):
|
26 |
-
def __init__(self,
|
27 |
-
width=1600,
|
28 |
-
height=1200,
|
29 |
-
name='PRT Renderer',
|
30 |
-
uv_mode=False,
|
31 |
-
ms_rate=1,
|
32 |
-
egl=False):
|
33 |
-
program_files = ['prt.vs', 'prt.fs'
|
34 |
-
] if not uv_mode else ['prt_uv.vs', 'prt_uv.fs']
|
35 |
-
CamRender.__init__(self,
|
36 |
-
width,
|
37 |
-
height,
|
38 |
-
name,
|
39 |
-
program_files=program_files,
|
40 |
-
color_size=8,
|
41 |
-
ms_rate=ms_rate,
|
42 |
-
egl=egl)
|
43 |
-
|
44 |
-
# WARNING: this differs from vertex_buffer and vertex_data in Render
|
45 |
-
self.vert_buffer = {}
|
46 |
-
self.vert_data = {}
|
47 |
-
|
48 |
-
self.vert_label_buffer = {}
|
49 |
-
self.vert_label_data = {}
|
50 |
-
|
51 |
-
self.norm_buffer = {}
|
52 |
-
self.norm_data = {}
|
53 |
-
|
54 |
-
self.tan_buffer = {}
|
55 |
-
self.tan_data = {}
|
56 |
-
|
57 |
-
self.btan_buffer = {}
|
58 |
-
self.btan_data = {}
|
59 |
-
|
60 |
-
self.prt1_buffer = {}
|
61 |
-
self.prt1_data = {}
|
62 |
-
|
63 |
-
self.prt2_buffer = {}
|
64 |
-
self.prt2_data = {}
|
65 |
-
|
66 |
-
self.prt3_buffer = {}
|
67 |
-
self.prt3_data = {}
|
68 |
-
|
69 |
-
self.uv_buffer = {}
|
70 |
-
self.uv_data = {}
|
71 |
-
|
72 |
-
self.render_texture_mat = {}
|
73 |
-
|
74 |
-
self.vertex_dim = {}
|
75 |
-
self.n_vertices = {}
|
76 |
-
self.label_dim = {}
|
77 |
-
|
78 |
-
self.norm_mat_unif = glGetUniformLocation(self.program, 'NormMat')
|
79 |
-
self.normalize_matrix = np.eye(4)
|
80 |
-
|
81 |
-
self.shcoeff_unif = glGetUniformLocation(self.program, 'SHCoeffs')
|
82 |
-
self.shcoeffs = np.zeros((9, 3))
|
83 |
-
self.shcoeffs[0, :] = 1.0
|
84 |
-
#self.shcoeffs[1:,:] = np.random.rand(8,3)
|
85 |
-
|
86 |
-
self.hasAlbedoUnif = glGetUniformLocation(self.program, 'hasAlbedoMap')
|
87 |
-
self.hasNormalUnif = glGetUniformLocation(self.program, 'hasNormalMap')
|
88 |
-
|
89 |
-
self.analyticUnif = glGetUniformLocation(self.program, 'analytic')
|
90 |
-
self.analytic = False
|
91 |
-
|
92 |
-
self.rot_mat_unif = glGetUniformLocation(self.program, 'RotMat')
|
93 |
-
self.rot_matrix = np.eye(3)
|
94 |
-
|
95 |
-
def set_texture(self, mat_name, smplr_name, texture):
|
96 |
-
# texture_image: H x W x 3
|
97 |
-
width = texture.shape[1]
|
98 |
-
height = texture.shape[0]
|
99 |
-
texture = np.flip(texture, 0)
|
100 |
-
img_data = np.fromstring(texture.tostring(), np.uint8)
|
101 |
-
|
102 |
-
if mat_name not in self.render_texture_mat:
|
103 |
-
self.render_texture_mat[mat_name] = {}
|
104 |
-
if smplr_name in self.render_texture_mat[mat_name].keys():
|
105 |
-
glDeleteTextures([self.render_texture_mat[mat_name][smplr_name]])
|
106 |
-
del self.render_texture_mat[mat_name][smplr_name]
|
107 |
-
|
108 |
-
self.render_texture_mat[mat_name][smplr_name] = glGenTextures(1)
|
109 |
-
glActiveTexture(GL_TEXTURE0)
|
110 |
-
|
111 |
-
glPixelStorei(GL_UNPACK_ALIGNMENT, 1)
|
112 |
-
glBindTexture(GL_TEXTURE_2D,
|
113 |
-
self.render_texture_mat[mat_name][smplr_name])
|
114 |
-
|
115 |
-
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_RGB,
|
116 |
-
GL_UNSIGNED_BYTE, img_data)
|
117 |
-
|
118 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAX_LEVEL, 3)
|
119 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
|
120 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
|
121 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
|
122 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
|
123 |
-
GL_LINEAR_MIPMAP_LINEAR)
|
124 |
-
|
125 |
-
glGenerateMipmap(GL_TEXTURE_2D)
|
126 |
-
|
127 |
-
def set_albedo(self, texture_image, mat_name='all'):
|
128 |
-
self.set_texture(mat_name, 'AlbedoMap', texture_image)
|
129 |
-
|
130 |
-
def set_normal_map(self, texture_image, mat_name='all'):
|
131 |
-
self.set_texture(mat_name, 'NormalMap', texture_image)
|
132 |
-
|
133 |
-
def set_mesh(self,
|
134 |
-
vertices,
|
135 |
-
faces,
|
136 |
-
norms,
|
137 |
-
faces_nml,
|
138 |
-
uvs,
|
139 |
-
faces_uvs,
|
140 |
-
prt,
|
141 |
-
faces_prt,
|
142 |
-
tans,
|
143 |
-
bitans,
|
144 |
-
verts_label=None,
|
145 |
-
mat_name='all'):
|
146 |
-
|
147 |
-
self.vert_data[mat_name] = vertices[faces.reshape([-1])]
|
148 |
-
self.vert_label_data[mat_name] = verts_label[faces.reshape([-1])]
|
149 |
-
self.n_vertices[mat_name] = self.vert_data[mat_name].shape[0]
|
150 |
-
self.vertex_dim[mat_name] = self.vert_data[mat_name].shape[1]
|
151 |
-
self.label_dim[mat_name] = self.vert_label_data[mat_name].shape[1]
|
152 |
-
|
153 |
-
if mat_name not in self.vert_buffer.keys():
|
154 |
-
self.vert_buffer[mat_name] = glGenBuffers(1)
|
155 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.vert_buffer[mat_name])
|
156 |
-
glBufferData(GL_ARRAY_BUFFER, self.vert_data[mat_name], GL_STATIC_DRAW)
|
157 |
-
|
158 |
-
if mat_name not in self.vert_label_buffer.keys():
|
159 |
-
self.vert_label_buffer[mat_name] = glGenBuffers(1)
|
160 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.vert_label_buffer[mat_name])
|
161 |
-
glBufferData(GL_ARRAY_BUFFER, self.vert_label_data[mat_name],
|
162 |
-
GL_STATIC_DRAW)
|
163 |
-
|
164 |
-
self.uv_data[mat_name] = uvs[faces_uvs.reshape([-1])]
|
165 |
-
if mat_name not in self.uv_buffer.keys():
|
166 |
-
self.uv_buffer[mat_name] = glGenBuffers(1)
|
167 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.uv_buffer[mat_name])
|
168 |
-
glBufferData(GL_ARRAY_BUFFER, self.uv_data[mat_name], GL_STATIC_DRAW)
|
169 |
-
|
170 |
-
self.norm_data[mat_name] = norms[faces_nml.reshape([-1])]
|
171 |
-
if mat_name not in self.norm_buffer.keys():
|
172 |
-
self.norm_buffer[mat_name] = glGenBuffers(1)
|
173 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer[mat_name])
|
174 |
-
glBufferData(GL_ARRAY_BUFFER, self.norm_data[mat_name], GL_STATIC_DRAW)
|
175 |
-
|
176 |
-
self.tan_data[mat_name] = tans[faces_nml.reshape([-1])]
|
177 |
-
if mat_name not in self.tan_buffer.keys():
|
178 |
-
self.tan_buffer[mat_name] = glGenBuffers(1)
|
179 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.tan_buffer[mat_name])
|
180 |
-
glBufferData(GL_ARRAY_BUFFER, self.tan_data[mat_name], GL_STATIC_DRAW)
|
181 |
-
|
182 |
-
self.btan_data[mat_name] = bitans[faces_nml.reshape([-1])]
|
183 |
-
if mat_name not in self.btan_buffer.keys():
|
184 |
-
self.btan_buffer[mat_name] = glGenBuffers(1)
|
185 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.btan_buffer[mat_name])
|
186 |
-
glBufferData(GL_ARRAY_BUFFER, self.btan_data[mat_name], GL_STATIC_DRAW)
|
187 |
-
|
188 |
-
self.prt1_data[mat_name] = prt[faces_prt.reshape([-1])][:, :3]
|
189 |
-
self.prt2_data[mat_name] = prt[faces_prt.reshape([-1])][:, 3:6]
|
190 |
-
self.prt3_data[mat_name] = prt[faces_prt.reshape([-1])][:, 6:]
|
191 |
-
|
192 |
-
if mat_name not in self.prt1_buffer.keys():
|
193 |
-
self.prt1_buffer[mat_name] = glGenBuffers(1)
|
194 |
-
if mat_name not in self.prt2_buffer.keys():
|
195 |
-
self.prt2_buffer[mat_name] = glGenBuffers(1)
|
196 |
-
if mat_name not in self.prt3_buffer.keys():
|
197 |
-
self.prt3_buffer[mat_name] = glGenBuffers(1)
|
198 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.prt1_buffer[mat_name])
|
199 |
-
glBufferData(GL_ARRAY_BUFFER, self.prt1_data[mat_name], GL_STATIC_DRAW)
|
200 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.prt2_buffer[mat_name])
|
201 |
-
glBufferData(GL_ARRAY_BUFFER, self.prt2_data[mat_name], GL_STATIC_DRAW)
|
202 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.prt3_buffer[mat_name])
|
203 |
-
glBufferData(GL_ARRAY_BUFFER, self.prt3_data[mat_name], GL_STATIC_DRAW)
|
204 |
-
|
205 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
206 |
-
|
207 |
-
def set_mesh_mtl(self,
|
208 |
-
vertices,
|
209 |
-
faces,
|
210 |
-
norms,
|
211 |
-
faces_nml,
|
212 |
-
uvs,
|
213 |
-
faces_uvs,
|
214 |
-
tans,
|
215 |
-
bitans,
|
216 |
-
prt,
|
217 |
-
verts_label=None):
|
218 |
-
for key in faces:
|
219 |
-
self.vert_data[key] = vertices[faces[key].reshape([-1])]
|
220 |
-
self.vert_label_data[key] = verts_label[faces[key].reshape([-1])]
|
221 |
-
self.n_vertices[key] = self.vert_data[key].shape[0]
|
222 |
-
self.vertex_dim[key] = self.vert_data[key].shape[1]
|
223 |
-
self.label_dim[key] = self.vert_label_data[key].shape[1]
|
224 |
-
|
225 |
-
if key not in self.vert_buffer.keys():
|
226 |
-
self.vert_buffer[key] = glGenBuffers(1)
|
227 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.vert_buffer[key])
|
228 |
-
glBufferData(GL_ARRAY_BUFFER, self.vert_data[key], GL_STATIC_DRAW)
|
229 |
-
|
230 |
-
if key not in self.vert_label_buffer.keys():
|
231 |
-
self.vert_label_buffer[key] = glGenBuffers(1)
|
232 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.vert_label_buffer[key])
|
233 |
-
glBufferData(GL_ARRAY_BUFFER, self.vert_label_data[key],
|
234 |
-
GL_STATIC_DRAW)
|
235 |
-
|
236 |
-
self.uv_data[key] = uvs[faces_uvs[key].reshape([-1])]
|
237 |
-
if key not in self.uv_buffer.keys():
|
238 |
-
self.uv_buffer[key] = glGenBuffers(1)
|
239 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.uv_buffer[key])
|
240 |
-
glBufferData(GL_ARRAY_BUFFER, self.uv_data[key], GL_STATIC_DRAW)
|
241 |
-
|
242 |
-
self.norm_data[key] = norms[faces_nml[key].reshape([-1])]
|
243 |
-
if key not in self.norm_buffer.keys():
|
244 |
-
self.norm_buffer[key] = glGenBuffers(1)
|
245 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer[key])
|
246 |
-
glBufferData(GL_ARRAY_BUFFER, self.norm_data[key], GL_STATIC_DRAW)
|
247 |
-
|
248 |
-
self.tan_data[key] = tans[faces_nml[key].reshape([-1])]
|
249 |
-
if key not in self.tan_buffer.keys():
|
250 |
-
self.tan_buffer[key] = glGenBuffers(1)
|
251 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.tan_buffer[key])
|
252 |
-
glBufferData(GL_ARRAY_BUFFER, self.tan_data[key], GL_STATIC_DRAW)
|
253 |
-
|
254 |
-
self.btan_data[key] = bitans[faces_nml[key].reshape([-1])]
|
255 |
-
if key not in self.btan_buffer.keys():
|
256 |
-
self.btan_buffer[key] = glGenBuffers(1)
|
257 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.btan_buffer[key])
|
258 |
-
glBufferData(GL_ARRAY_BUFFER, self.btan_data[key], GL_STATIC_DRAW)
|
259 |
-
|
260 |
-
self.prt1_data[key] = prt[faces[key].reshape([-1])][:, :3]
|
261 |
-
self.prt2_data[key] = prt[faces[key].reshape([-1])][:, 3:6]
|
262 |
-
self.prt3_data[key] = prt[faces[key].reshape([-1])][:, 6:]
|
263 |
-
|
264 |
-
if key not in self.prt1_buffer.keys():
|
265 |
-
self.prt1_buffer[key] = glGenBuffers(1)
|
266 |
-
if key not in self.prt2_buffer.keys():
|
267 |
-
self.prt2_buffer[key] = glGenBuffers(1)
|
268 |
-
if key not in self.prt3_buffer.keys():
|
269 |
-
self.prt3_buffer[key] = glGenBuffers(1)
|
270 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.prt1_buffer[key])
|
271 |
-
glBufferData(GL_ARRAY_BUFFER, self.prt1_data[key], GL_STATIC_DRAW)
|
272 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.prt2_buffer[key])
|
273 |
-
glBufferData(GL_ARRAY_BUFFER, self.prt2_data[key], GL_STATIC_DRAW)
|
274 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.prt3_buffer[key])
|
275 |
-
glBufferData(GL_ARRAY_BUFFER, self.prt3_data[key], GL_STATIC_DRAW)
|
276 |
-
|
277 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
278 |
-
|
279 |
-
def cleanup(self):
|
280 |
-
|
281 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
282 |
-
for key in self.vert_data:
|
283 |
-
glDeleteBuffers(1, [self.vert_buffer[key]])
|
284 |
-
glDeleteBuffers(1, [self.norm_buffer[key]])
|
285 |
-
glDeleteBuffers(1, [self.uv_buffer[key]])
|
286 |
-
glDeleteBuffers(1, [self.vert_label_buffer[key]])
|
287 |
-
|
288 |
-
glDeleteBuffers(1, [self.tan_buffer[key]])
|
289 |
-
glDeleteBuffers(1, [self.btan_buffer[key]])
|
290 |
-
glDeleteBuffers(1, [self.prt1_buffer[key]])
|
291 |
-
glDeleteBuffers(1, [self.prt2_buffer[key]])
|
292 |
-
glDeleteBuffers(1, [self.prt3_buffer[key]])
|
293 |
-
|
294 |
-
glDeleteBuffers(1, [])
|
295 |
-
|
296 |
-
for smplr in self.render_texture_mat[key]:
|
297 |
-
glDeleteTextures([self.render_texture_mat[key][smplr]])
|
298 |
-
|
299 |
-
self.vert_buffer = {}
|
300 |
-
self.vert_data = {}
|
301 |
-
|
302 |
-
self.vert_label_buffer = {}
|
303 |
-
self.vert_label_data = {}
|
304 |
-
|
305 |
-
self.norm_buffer = {}
|
306 |
-
self.norm_data = {}
|
307 |
-
|
308 |
-
self.tan_buffer = {}
|
309 |
-
self.tan_data = {}
|
310 |
-
|
311 |
-
self.btan_buffer = {}
|
312 |
-
self.btan_data = {}
|
313 |
-
|
314 |
-
self.prt1_buffer = {}
|
315 |
-
self.prt1_data = {}
|
316 |
-
|
317 |
-
self.prt2_buffer = {}
|
318 |
-
self.prt2_data = {}
|
319 |
-
|
320 |
-
self.prt3_buffer = {}
|
321 |
-
self.prt3_data = {}
|
322 |
-
|
323 |
-
self.uv_buffer = {}
|
324 |
-
self.uv_data = {}
|
325 |
-
|
326 |
-
self.render_texture_mat = {}
|
327 |
-
|
328 |
-
self.vertex_dim = {}
|
329 |
-
self.n_vertices = {}
|
330 |
-
self.label_dim = {}
|
331 |
-
|
332 |
-
def randomize_sh(self):
|
333 |
-
self.shcoeffs[0, :] = 0.8
|
334 |
-
self.shcoeffs[1:, :] = 1.0 * np.random.rand(8, 3)
|
335 |
-
|
336 |
-
def set_sh(self, sh):
|
337 |
-
self.shcoeffs = sh
|
338 |
-
|
339 |
-
def set_norm_mat(self, scale, center):
|
340 |
-
N = np.eye(4)
|
341 |
-
N[:3, :3] = scale * np.eye(3)
|
342 |
-
N[:3, 3] = -scale * center
|
343 |
-
|
344 |
-
self.normalize_matrix = N
|
345 |
-
|
346 |
-
def draw(self):
|
347 |
-
self.draw_init()
|
348 |
-
|
349 |
-
glDisable(GL_BLEND)
|
350 |
-
#glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
|
351 |
-
glEnable(GL_MULTISAMPLE)
|
352 |
-
|
353 |
-
glUseProgram(self.program)
|
354 |
-
glUniformMatrix4fv(self.norm_mat_unif, 1, GL_FALSE,
|
355 |
-
self.normalize_matrix.transpose())
|
356 |
-
glUniformMatrix4fv(self.model_mat_unif, 1, GL_FALSE,
|
357 |
-
self.model_view_matrix.transpose())
|
358 |
-
glUniformMatrix4fv(self.persp_mat_unif, 1, GL_FALSE,
|
359 |
-
self.projection_matrix.transpose())
|
360 |
-
|
361 |
-
if 'AlbedoMap' in self.render_texture_mat['all']:
|
362 |
-
glUniform1ui(self.hasAlbedoUnif, GLuint(1))
|
363 |
-
else:
|
364 |
-
glUniform1ui(self.hasAlbedoUnif, GLuint(0))
|
365 |
-
|
366 |
-
if 'NormalMap' in self.render_texture_mat['all']:
|
367 |
-
glUniform1ui(self.hasNormalUnif, GLuint(1))
|
368 |
-
else:
|
369 |
-
glUniform1ui(self.hasNormalUnif, GLuint(0))
|
370 |
-
|
371 |
-
glUniform1ui(self.analyticUnif,
|
372 |
-
GLuint(1) if self.analytic else GLuint(0))
|
373 |
-
|
374 |
-
glUniform3fv(self.shcoeff_unif, 9, self.shcoeffs)
|
375 |
-
|
376 |
-
glUniformMatrix3fv(self.rot_mat_unif, 1, GL_FALSE,
|
377 |
-
self.rot_matrix.transpose())
|
378 |
-
|
379 |
-
for mat in self.vert_buffer:
|
380 |
-
# Handle vertex buffer
|
381 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.vert_buffer[mat])
|
382 |
-
glEnableVertexAttribArray(0)
|
383 |
-
glVertexAttribPointer(0, self.vertex_dim[mat], GL_DOUBLE, GL_FALSE,
|
384 |
-
0, None)
|
385 |
-
|
386 |
-
# Handle normal buffer
|
387 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.norm_buffer[mat])
|
388 |
-
glEnableVertexAttribArray(1)
|
389 |
-
glVertexAttribPointer(1, 3, GL_DOUBLE, GL_FALSE, 0, None)
|
390 |
-
|
391 |
-
# Handle uv buffer
|
392 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.uv_buffer[mat])
|
393 |
-
glEnableVertexAttribArray(2)
|
394 |
-
glVertexAttribPointer(2, 2, GL_DOUBLE, GL_FALSE, 0, None)
|
395 |
-
|
396 |
-
# Handle tan buffer
|
397 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.tan_buffer[mat])
|
398 |
-
glEnableVertexAttribArray(3)
|
399 |
-
glVertexAttribPointer(3, 3, GL_DOUBLE, GL_FALSE, 0, None)
|
400 |
-
|
401 |
-
# Handle btan buffer
|
402 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.btan_buffer[mat])
|
403 |
-
glEnableVertexAttribArray(4)
|
404 |
-
glVertexAttribPointer(4, 3, GL_DOUBLE, GL_FALSE, 0, None)
|
405 |
-
|
406 |
-
# Handle PTR buffer
|
407 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.prt1_buffer[mat])
|
408 |
-
glEnableVertexAttribArray(5)
|
409 |
-
glVertexAttribPointer(5, 3, GL_DOUBLE, GL_FALSE, 0, None)
|
410 |
-
|
411 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.prt2_buffer[mat])
|
412 |
-
glEnableVertexAttribArray(6)
|
413 |
-
glVertexAttribPointer(6, 3, GL_DOUBLE, GL_FALSE, 0, None)
|
414 |
-
|
415 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.prt3_buffer[mat])
|
416 |
-
glEnableVertexAttribArray(7)
|
417 |
-
glVertexAttribPointer(7, 3, GL_DOUBLE, GL_FALSE, 0, None)
|
418 |
-
|
419 |
-
# Handle vertex label buffer
|
420 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.vert_label_buffer[mat])
|
421 |
-
glEnableVertexAttribArray(8)
|
422 |
-
glVertexAttribPointer(8, self.label_dim[mat], GL_DOUBLE, GL_FALSE,
|
423 |
-
0, None)
|
424 |
-
|
425 |
-
for i, smplr in enumerate(self.render_texture_mat[mat]):
|
426 |
-
glActiveTexture(GL_TEXTURE0 + i)
|
427 |
-
glBindTexture(GL_TEXTURE_2D,
|
428 |
-
self.render_texture_mat[mat][smplr])
|
429 |
-
glUniform1i(glGetUniformLocation(self.program, smplr), i)
|
430 |
-
|
431 |
-
glDrawArrays(GL_TRIANGLES, 0, self.n_vertices[mat])
|
432 |
-
|
433 |
-
glDisableVertexAttribArray(8)
|
434 |
-
glDisableVertexAttribArray(7)
|
435 |
-
glDisableVertexAttribArray(6)
|
436 |
-
glDisableVertexAttribArray(5)
|
437 |
-
glDisableVertexAttribArray(4)
|
438 |
-
glDisableVertexAttribArray(3)
|
439 |
-
glDisableVertexAttribArray(2)
|
440 |
-
glDisableVertexAttribArray(1)
|
441 |
-
glDisableVertexAttribArray(0)
|
442 |
-
|
443 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
444 |
-
|
445 |
-
glUseProgram(0)
|
446 |
-
|
447 |
-
glDisable(GL_BLEND)
|
448 |
-
glDisable(GL_MULTISAMPLE)
|
449 |
-
|
450 |
-
self.draw_end()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer / gl /render2.py
DELETED
@@ -1,384 +0,0 @@
|
|
1 |
-
'''
|
2 |
-
MIT License
|
3 |
-
Copyright (c) 2019 Shunsuke Saito, Zeng Huang, and Ryota Natsume
|
4 |
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
5 |
-
of this software and associated documentation files (the "Software"), to deal
|
6 |
-
in the Software without restriction, including without limitation the rights
|
7 |
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
8 |
-
copies of the Software, and to permit persons to whom the Software is
|
9 |
-
furnished to do so, subject to the following conditions:
|
10 |
-
The above copyright notice and this permission notice shall be included in all
|
11 |
-
copies or substantial portions of the Software.
|
12 |
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
13 |
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
14 |
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
15 |
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
16 |
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
17 |
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
18 |
-
SOFTWARE.
|
19 |
-
'''
|
20 |
-
import numpy as np
|
21 |
-
from OpenGL.GLUT import *
|
22 |
-
from .framework import *
|
23 |
-
|
24 |
-
_glut_window = None
|
25 |
-
|
26 |
-
|
27 |
-
class Render:
|
28 |
-
def __init__(self,
|
29 |
-
width=1600,
|
30 |
-
height=1200,
|
31 |
-
name='GL Renderer',
|
32 |
-
program_files=['simple.fs', 'simple.vs'],
|
33 |
-
color_size=1,
|
34 |
-
ms_rate=1):
|
35 |
-
self.width = width
|
36 |
-
self.height = height
|
37 |
-
self.name = name
|
38 |
-
self.display_mode = GLUT_DOUBLE | GLUT_RGB | GLUT_DEPTH
|
39 |
-
self.use_inverse_depth = False
|
40 |
-
|
41 |
-
global _glut_window
|
42 |
-
if _glut_window is None:
|
43 |
-
glutInit()
|
44 |
-
glutInitDisplayMode(self.display_mode)
|
45 |
-
glutInitWindowSize(self.width, self.height)
|
46 |
-
glutInitWindowPosition(0, 0)
|
47 |
-
_glut_window = glutCreateWindow("My Render.")
|
48 |
-
|
49 |
-
# glEnable(GL_DEPTH_CLAMP)
|
50 |
-
glEnable(GL_DEPTH_TEST)
|
51 |
-
|
52 |
-
glClampColor(GL_CLAMP_READ_COLOR, GL_FALSE)
|
53 |
-
glClampColor(GL_CLAMP_FRAGMENT_COLOR, GL_FALSE)
|
54 |
-
glClampColor(GL_CLAMP_VERTEX_COLOR, GL_FALSE)
|
55 |
-
|
56 |
-
# init program
|
57 |
-
shader_list = []
|
58 |
-
|
59 |
-
for program_file in program_files:
|
60 |
-
_, ext = os.path.splitext(program_file)
|
61 |
-
if ext == '.vs':
|
62 |
-
shader_list.append(loadShader(GL_VERTEX_SHADER, program_file))
|
63 |
-
elif ext == '.fs':
|
64 |
-
shader_list.append(loadShader(GL_FRAGMENT_SHADER,
|
65 |
-
program_file))
|
66 |
-
elif ext == '.gs':
|
67 |
-
shader_list.append(loadShader(GL_GEOMETRY_SHADER,
|
68 |
-
program_file))
|
69 |
-
|
70 |
-
self.program = createProgram(shader_list)
|
71 |
-
|
72 |
-
for shader in shader_list:
|
73 |
-
glDeleteShader(shader)
|
74 |
-
|
75 |
-
# Init uniform variables
|
76 |
-
self.model_mat_unif = glGetUniformLocation(self.program, 'ModelMat')
|
77 |
-
self.persp_mat_unif = glGetUniformLocation(self.program, 'PerspMat')
|
78 |
-
|
79 |
-
self.vertex_buffer = glGenBuffers(1)
|
80 |
-
|
81 |
-
# Init screen quad program and buffer
|
82 |
-
self.quad_program, self.quad_buffer = self.init_quad_program()
|
83 |
-
|
84 |
-
# Configure frame buffer
|
85 |
-
self.frame_buffer = glGenFramebuffers(1)
|
86 |
-
glBindFramebuffer(GL_FRAMEBUFFER, self.frame_buffer)
|
87 |
-
|
88 |
-
self.intermediate_fbo = None
|
89 |
-
if ms_rate > 1:
|
90 |
-
# Configure texture buffer to render to
|
91 |
-
self.color_buffer = []
|
92 |
-
for i in range(color_size):
|
93 |
-
color_buffer = glGenTextures(1)
|
94 |
-
multi_sample_rate = ms_rate
|
95 |
-
glBindTexture(GL_TEXTURE_2D_MULTISAMPLE, color_buffer)
|
96 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,
|
97 |
-
GL_CLAMP_TO_EDGE)
|
98 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,
|
99 |
-
GL_CLAMP_TO_EDGE)
|
100 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
|
101 |
-
GL_LINEAR)
|
102 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
|
103 |
-
GL_LINEAR)
|
104 |
-
glTexImage2DMultisample(GL_TEXTURE_2D_MULTISAMPLE,
|
105 |
-
multi_sample_rate, GL_RGBA32F,
|
106 |
-
self.width, self.height, GL_TRUE)
|
107 |
-
glBindTexture(GL_TEXTURE_2D_MULTISAMPLE, 0)
|
108 |
-
glFramebufferTexture2D(GL_FRAMEBUFFER,
|
109 |
-
GL_COLOR_ATTACHMENT0 + i,
|
110 |
-
GL_TEXTURE_2D_MULTISAMPLE, color_buffer,
|
111 |
-
0)
|
112 |
-
self.color_buffer.append(color_buffer)
|
113 |
-
|
114 |
-
self.render_buffer = glGenRenderbuffers(1)
|
115 |
-
glBindRenderbuffer(GL_RENDERBUFFER, self.render_buffer)
|
116 |
-
glRenderbufferStorageMultisample(GL_RENDERBUFFER,
|
117 |
-
multi_sample_rate,
|
118 |
-
GL_DEPTH24_STENCIL8, self.width,
|
119 |
-
self.height)
|
120 |
-
glBindRenderbuffer(GL_RENDERBUFFER, 0)
|
121 |
-
glFramebufferRenderbuffer(GL_FRAMEBUFFER,
|
122 |
-
GL_DEPTH_STENCIL_ATTACHMENT,
|
123 |
-
GL_RENDERBUFFER, self.render_buffer)
|
124 |
-
|
125 |
-
attachments = []
|
126 |
-
for i in range(color_size):
|
127 |
-
attachments.append(GL_COLOR_ATTACHMENT0 + i)
|
128 |
-
glDrawBuffers(color_size, attachments)
|
129 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
130 |
-
|
131 |
-
self.intermediate_fbo = glGenFramebuffers(1)
|
132 |
-
glBindFramebuffer(GL_FRAMEBUFFER, self.intermediate_fbo)
|
133 |
-
|
134 |
-
self.screen_texture = []
|
135 |
-
for i in range(color_size):
|
136 |
-
screen_texture = glGenTextures(1)
|
137 |
-
glBindTexture(GL_TEXTURE_2D, screen_texture)
|
138 |
-
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, self.width,
|
139 |
-
self.height, 0, GL_RGBA, GL_FLOAT, None)
|
140 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
|
141 |
-
GL_LINEAR)
|
142 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
|
143 |
-
GL_LINEAR)
|
144 |
-
glFramebufferTexture2D(GL_FRAMEBUFFER,
|
145 |
-
GL_COLOR_ATTACHMENT0 + i, GL_TEXTURE_2D,
|
146 |
-
screen_texture, 0)
|
147 |
-
self.screen_texture.append(screen_texture)
|
148 |
-
|
149 |
-
glDrawBuffers(color_size, attachments)
|
150 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
151 |
-
else:
|
152 |
-
self.color_buffer = []
|
153 |
-
for i in range(color_size):
|
154 |
-
color_buffer = glGenTextures(1)
|
155 |
-
glBindTexture(GL_TEXTURE_2D, color_buffer)
|
156 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,
|
157 |
-
GL_CLAMP_TO_EDGE)
|
158 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,
|
159 |
-
GL_CLAMP_TO_EDGE)
|
160 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
|
161 |
-
GL_NEAREST)
|
162 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
|
163 |
-
GL_NEAREST)
|
164 |
-
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA32F, self.width,
|
165 |
-
self.height, 0, GL_RGBA, GL_FLOAT, None)
|
166 |
-
glFramebufferTexture2D(GL_FRAMEBUFFER,
|
167 |
-
GL_COLOR_ATTACHMENT0 + i, GL_TEXTURE_2D,
|
168 |
-
color_buffer, 0)
|
169 |
-
self.color_buffer.append(color_buffer)
|
170 |
-
|
171 |
-
# Configure depth texture map to render to
|
172 |
-
self.depth_buffer = glGenTextures(1)
|
173 |
-
glBindTexture(GL_TEXTURE_2D, self.depth_buffer)
|
174 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT)
|
175 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT)
|
176 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
|
177 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
|
178 |
-
glTexParameteri(GL_TEXTURE_2D, GL_DEPTH_TEXTURE_MODE, GL_INTENSITY)
|
179 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_MODE,
|
180 |
-
GL_COMPARE_R_TO_TEXTURE)
|
181 |
-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FUNC, GL_LEQUAL)
|
182 |
-
glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT, self.width,
|
183 |
-
self.height, 0, GL_DEPTH_COMPONENT, GL_FLOAT, None)
|
184 |
-
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT,
|
185 |
-
GL_TEXTURE_2D, self.depth_buffer, 0)
|
186 |
-
|
187 |
-
attachments = []
|
188 |
-
for i in range(color_size):
|
189 |
-
attachments.append(GL_COLOR_ATTACHMENT0 + i)
|
190 |
-
glDrawBuffers(color_size, attachments)
|
191 |
-
self.screen_texture = self.color_buffer
|
192 |
-
|
193 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
194 |
-
|
195 |
-
# Configure texture buffer if needed
|
196 |
-
self.render_texture = None
|
197 |
-
|
198 |
-
# NOTE: original render_texture only support one input
|
199 |
-
# this is tentative member of this issue
|
200 |
-
self.render_texture_v2 = {}
|
201 |
-
|
202 |
-
# Inner storage for buffer data
|
203 |
-
self.vertex_data = None
|
204 |
-
self.vertex_dim = None
|
205 |
-
self.n_vertices = None
|
206 |
-
|
207 |
-
self.model_view_matrix = None
|
208 |
-
self.projection_matrix = None
|
209 |
-
|
210 |
-
glutDisplayFunc(self.display)
|
211 |
-
|
212 |
-
def init_quad_program(self):
|
213 |
-
shader_list = []
|
214 |
-
|
215 |
-
shader_list.append(loadShader(GL_VERTEX_SHADER, "quad.vs"))
|
216 |
-
shader_list.append(loadShader(GL_FRAGMENT_SHADER, "quad.fs"))
|
217 |
-
|
218 |
-
the_program = createProgram(shader_list)
|
219 |
-
|
220 |
-
for shader in shader_list:
|
221 |
-
glDeleteShader(shader)
|
222 |
-
|
223 |
-
# vertex attributes for a quad that fills the entire screen in Normalized Device Coordinates.
|
224 |
-
# positions # texCoords
|
225 |
-
quad_vertices = np.array([
|
226 |
-
-1.0, 1.0, 0.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0, -1.0, 1.0, 0.0,
|
227 |
-
-1.0, 1.0, 0.0, 1.0, 1.0, -1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0
|
228 |
-
])
|
229 |
-
|
230 |
-
quad_buffer = glGenBuffers(1)
|
231 |
-
glBindBuffer(GL_ARRAY_BUFFER, quad_buffer)
|
232 |
-
glBufferData(GL_ARRAY_BUFFER, quad_vertices, GL_STATIC_DRAW)
|
233 |
-
|
234 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
235 |
-
|
236 |
-
return the_program, quad_buffer
|
237 |
-
|
238 |
-
def set_mesh(self, vertices, faces):
|
239 |
-
self.vertex_data = vertices[faces.reshape([-1])]
|
240 |
-
self.vertex_dim = self.vertex_data.shape[1]
|
241 |
-
self.n_vertices = self.vertex_data.shape[0]
|
242 |
-
|
243 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.vertex_buffer)
|
244 |
-
glBufferData(GL_ARRAY_BUFFER, self.vertex_data, GL_STATIC_DRAW)
|
245 |
-
|
246 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
247 |
-
|
248 |
-
def set_viewpoint(self, projection, model_view):
|
249 |
-
self.projection_matrix = projection
|
250 |
-
self.model_view_matrix = model_view
|
251 |
-
|
252 |
-
def draw_init(self):
|
253 |
-
glBindFramebuffer(GL_FRAMEBUFFER, self.frame_buffer)
|
254 |
-
glEnable(GL_DEPTH_TEST)
|
255 |
-
|
256 |
-
# glClearColor(0.0, 0.0, 0.0, 0.0)
|
257 |
-
glClearColor(1.0, 1.0, 1.0, 0.0) # Black background
|
258 |
-
|
259 |
-
if self.use_inverse_depth:
|
260 |
-
glDepthFunc(GL_GREATER)
|
261 |
-
glClearDepth(0.0)
|
262 |
-
else:
|
263 |
-
glDepthFunc(GL_LESS)
|
264 |
-
glClearDepth(1.0)
|
265 |
-
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
|
266 |
-
|
267 |
-
def draw_end(self):
|
268 |
-
if self.intermediate_fbo is not None:
|
269 |
-
for i in range(len(self.color_buffer)):
|
270 |
-
glBindFramebuffer(GL_READ_FRAMEBUFFER, self.frame_buffer)
|
271 |
-
glReadBuffer(GL_COLOR_ATTACHMENT0 + i)
|
272 |
-
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self.intermediate_fbo)
|
273 |
-
glDrawBuffer(GL_COLOR_ATTACHMENT0 + i)
|
274 |
-
glBlitFramebuffer(0, 0, self.width, self.height, 0, 0,
|
275 |
-
self.width, self.height, GL_COLOR_BUFFER_BIT,
|
276 |
-
GL_NEAREST)
|
277 |
-
|
278 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
279 |
-
glDepthFunc(GL_LESS)
|
280 |
-
glClearDepth(1.0)
|
281 |
-
|
282 |
-
def draw(self):
|
283 |
-
self.draw_init()
|
284 |
-
|
285 |
-
glUseProgram(self.program)
|
286 |
-
glUniformMatrix4fv(self.model_mat_unif, 1, GL_FALSE,
|
287 |
-
self.model_view_matrix.transpose())
|
288 |
-
glUniformMatrix4fv(self.persp_mat_unif, 1, GL_FALSE,
|
289 |
-
self.projection_matrix.transpose())
|
290 |
-
|
291 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.vertex_buffer)
|
292 |
-
|
293 |
-
glEnableVertexAttribArray(0)
|
294 |
-
glVertexAttribPointer(0, self.vertex_dim, GL_DOUBLE, GL_FALSE, 0, None)
|
295 |
-
|
296 |
-
glDrawArrays(GL_TRIANGLES, 0, self.n_vertices)
|
297 |
-
|
298 |
-
glDisableVertexAttribArray(0)
|
299 |
-
|
300 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
301 |
-
|
302 |
-
glUseProgram(0)
|
303 |
-
|
304 |
-
self.draw_end()
|
305 |
-
|
306 |
-
def get_color(self, color_id=0):
|
307 |
-
glBindFramebuffer(
|
308 |
-
GL_FRAMEBUFFER, self.intermediate_fbo
|
309 |
-
if self.intermediate_fbo is not None else self.frame_buffer)
|
310 |
-
glReadBuffer(GL_COLOR_ATTACHMENT0 + color_id)
|
311 |
-
data = glReadPixels(0,
|
312 |
-
0,
|
313 |
-
self.width,
|
314 |
-
self.height,
|
315 |
-
GL_RGBA,
|
316 |
-
GL_FLOAT,
|
317 |
-
outputType=None)
|
318 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
319 |
-
rgb = data.reshape(self.height, self.width, -1)
|
320 |
-
rgb = np.flip(rgb, 0)
|
321 |
-
return rgb
|
322 |
-
|
323 |
-
def get_z_value(self):
|
324 |
-
glBindFramebuffer(GL_FRAMEBUFFER, self.frame_buffer)
|
325 |
-
data = glReadPixels(0,
|
326 |
-
0,
|
327 |
-
self.width,
|
328 |
-
self.height,
|
329 |
-
GL_DEPTH_COMPONENT,
|
330 |
-
GL_FLOAT,
|
331 |
-
outputType=None)
|
332 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
333 |
-
z = data.reshape(self.height, self.width)
|
334 |
-
z = np.flip(z, 0)
|
335 |
-
return z
|
336 |
-
|
337 |
-
def display(self):
|
338 |
-
# First we draw a scene.
|
339 |
-
# Notice the result is stored in the texture buffer.
|
340 |
-
self.draw()
|
341 |
-
|
342 |
-
# Then we return to the default frame buffer since we will display on the screen.
|
343 |
-
glBindFramebuffer(GL_FRAMEBUFFER, 0)
|
344 |
-
|
345 |
-
# Do the clean-up.
|
346 |
-
# glClearColor(0.0, 0.0, 0.0, 0.0) #Black background
|
347 |
-
glClearColor(1.0, 1.0, 1.0, 0.0) # Black background
|
348 |
-
glClear(GL_COLOR_BUFFER_BIT)
|
349 |
-
|
350 |
-
# We draw a rectangle which covers the whole screen.
|
351 |
-
glUseProgram(self.quad_program)
|
352 |
-
glBindBuffer(GL_ARRAY_BUFFER, self.quad_buffer)
|
353 |
-
|
354 |
-
size_of_double = 8
|
355 |
-
glEnableVertexAttribArray(0)
|
356 |
-
glVertexAttribPointer(0, 2, GL_DOUBLE, GL_FALSE, 4 * size_of_double,
|
357 |
-
None)
|
358 |
-
glEnableVertexAttribArray(1)
|
359 |
-
glVertexAttribPointer(1, 2, GL_DOUBLE, GL_FALSE, 4 * size_of_double,
|
360 |
-
c_void_p(2 * size_of_double))
|
361 |
-
|
362 |
-
glDisable(GL_DEPTH_TEST)
|
363 |
-
|
364 |
-
# The stored texture is then mapped to this rectangle.
|
365 |
-
# properly assing color buffer texture
|
366 |
-
glActiveTexture(GL_TEXTURE0)
|
367 |
-
glBindTexture(GL_TEXTURE_2D, self.screen_texture[0])
|
368 |
-
glUniform1i(glGetUniformLocation(self.quad_program, 'screenTexture'),
|
369 |
-
0)
|
370 |
-
|
371 |
-
glDrawArrays(GL_TRIANGLES, 0, 6)
|
372 |
-
|
373 |
-
glDisableVertexAttribArray(1)
|
374 |
-
glDisableVertexAttribArray(0)
|
375 |
-
|
376 |
-
glEnable(GL_DEPTH_TEST)
|
377 |
-
glBindBuffer(GL_ARRAY_BUFFER, 0)
|
378 |
-
glUseProgram(0)
|
379 |
-
|
380 |
-
glutSwapBuffers()
|
381 |
-
glutPostRedisplay()
|
382 |
-
|
383 |
-
def show(self):
|
384 |
-
glutMainLoop()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer /glm.py
DELETED
@@ -1,143 +0,0 @@
|
|
1 |
-
|
2 |
-
# -*- coding: utf-8 -*-
|
3 |
-
|
4 |
-
# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
|
5 |
-
# holder of all proprietary rights on this computer program.
|
6 |
-
# You can only use this computer program if you have closed
|
7 |
-
# a license agreement with MPG or you get the right to use the computer
|
8 |
-
# program from someone who is authorized to grant you that right.
|
9 |
-
# Any use of the computer program without a valid license is prohibited and
|
10 |
-
# liable to prosecution.
|
11 |
-
#
|
12 |
-
# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
|
13 |
-
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
|
14 |
-
# for Intelligent Systems. All rights reserved.
|
15 |
-
#
|
16 |
-
# Contact: [email protected]
|
17 |
-
|
18 |
-
import numpy as np
|
19 |
-
|
20 |
-
|
21 |
-
def vec3(x, y, z):
|
22 |
-
return np.array([x, y, z], dtype=np.float32)
|
23 |
-
|
24 |
-
|
25 |
-
def radians(v):
|
26 |
-
return np.radians(v)
|
27 |
-
|
28 |
-
|
29 |
-
def identity():
|
30 |
-
return np.identity(4, dtype=np.float32)
|
31 |
-
|
32 |
-
|
33 |
-
def empty():
|
34 |
-
return np.zeros([4, 4], dtype=np.float32)
|
35 |
-
|
36 |
-
|
37 |
-
def magnitude(v):
|
38 |
-
return np.linalg.norm(v)
|
39 |
-
|
40 |
-
|
41 |
-
def normalize(v):
|
42 |
-
m = magnitude(v)
|
43 |
-
return v if m == 0 else v / m
|
44 |
-
|
45 |
-
|
46 |
-
def dot(u, v):
|
47 |
-
return np.sum(u * v)
|
48 |
-
|
49 |
-
|
50 |
-
def cross(u, v):
|
51 |
-
res = vec3(0, 0, 0)
|
52 |
-
res[0] = u[1] * v[2] - u[2] * v[1]
|
53 |
-
res[1] = u[2] * v[0] - u[0] * v[2]
|
54 |
-
res[2] = u[0] * v[1] - u[1] * v[0]
|
55 |
-
return res
|
56 |
-
|
57 |
-
|
58 |
-
# below functions can be optimized
|
59 |
-
|
60 |
-
|
61 |
-
def translate(m, v):
|
62 |
-
res = np.copy(m)
|
63 |
-
res[:, 3] = m[:, 0] * v[0] + m[:, 1] * v[1] + m[:, 2] * v[2] + m[:, 3]
|
64 |
-
return res
|
65 |
-
|
66 |
-
|
67 |
-
def rotate(m, angle, v):
|
68 |
-
a = angle
|
69 |
-
c = np.cos(a)
|
70 |
-
s = np.sin(a)
|
71 |
-
|
72 |
-
axis = normalize(v)
|
73 |
-
temp = (1 - c) * axis
|
74 |
-
|
75 |
-
rot = empty()
|
76 |
-
rot[0][0] = c + temp[0] * axis[0]
|
77 |
-
rot[0][1] = temp[0] * axis[1] + s * axis[2]
|
78 |
-
rot[0][2] = temp[0] * axis[2] - s * axis[1]
|
79 |
-
|
80 |
-
rot[1][0] = temp[1] * axis[0] - s * axis[2]
|
81 |
-
rot[1][1] = c + temp[1] * axis[1]
|
82 |
-
rot[1][2] = temp[1] * axis[2] + s * axis[0]
|
83 |
-
|
84 |
-
rot[2][0] = temp[2] * axis[0] + s * axis[1]
|
85 |
-
rot[2][1] = temp[2] * axis[1] - s * axis[0]
|
86 |
-
rot[2][2] = c + temp[2] * axis[2]
|
87 |
-
|
88 |
-
res = empty()
|
89 |
-
res[:, 0] = m[:, 0] * rot[0][0] + m[:, 1] * rot[0][1] + m[:, 2] * rot[0][2]
|
90 |
-
res[:, 1] = m[:, 0] * rot[1][0] + m[:, 1] * rot[1][1] + m[:, 2] * rot[1][2]
|
91 |
-
res[:, 2] = m[:, 0] * rot[2][0] + m[:, 1] * rot[2][1] + m[:, 2] * rot[2][2]
|
92 |
-
res[:, 3] = m[:, 3]
|
93 |
-
return res
|
94 |
-
|
95 |
-
|
96 |
-
def perspective(fovy, aspect, zNear, zFar):
|
97 |
-
tanHalfFovy = np.tan(fovy / 2)
|
98 |
-
|
99 |
-
res = empty()
|
100 |
-
res[0][0] = 1 / (aspect * tanHalfFovy)
|
101 |
-
res[1][1] = 1 / (tanHalfFovy)
|
102 |
-
res[2][3] = -1
|
103 |
-
res[2][2] = -(zFar + zNear) / (zFar - zNear)
|
104 |
-
res[3][2] = -(2 * zFar * zNear) / (zFar - zNear)
|
105 |
-
|
106 |
-
return res.T
|
107 |
-
|
108 |
-
|
109 |
-
def ortho(left, right, bottom, top, zNear, zFar):
|
110 |
-
# res = np.ones([4, 4], dtype=np.float32)
|
111 |
-
res = identity()
|
112 |
-
res[0][0] = 2 / (right - left)
|
113 |
-
res[1][1] = 2 / (top - bottom)
|
114 |
-
res[2][2] = -2 / (zFar - zNear)
|
115 |
-
res[3][0] = -(right + left) / (right - left)
|
116 |
-
res[3][1] = -(top + bottom) / (top - bottom)
|
117 |
-
res[3][2] = -(zFar + zNear) / (zFar - zNear)
|
118 |
-
return res.T
|
119 |
-
|
120 |
-
|
121 |
-
def lookat(eye, center, up):
|
122 |
-
f = normalize(center - eye)
|
123 |
-
s = normalize(cross(f, up))
|
124 |
-
u = cross(s, f)
|
125 |
-
|
126 |
-
res = identity()
|
127 |
-
res[0][0] = s[0]
|
128 |
-
res[1][0] = s[1]
|
129 |
-
res[2][0] = s[2]
|
130 |
-
res[0][1] = u[0]
|
131 |
-
res[1][1] = u[1]
|
132 |
-
res[2][1] = u[2]
|
133 |
-
res[0][2] = -f[0]
|
134 |
-
res[1][2] = -f[1]
|
135 |
-
res[2][2] = -f[2]
|
136 |
-
res[3][0] = -dot(s, eye)
|
137 |
-
res[3][1] = -dot(u, eye)
|
138 |
-
res[3][2] = -dot(f, eye)
|
139 |
-
return res.T
|
140 |
-
|
141 |
-
|
142 |
-
def transform(d, m):
|
143 |
-
return np.dot(m, d.T).T
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer /mesh.py
DELETED
@@ -1,526 +0,0 @@
|
|
1 |
-
|
2 |
-
# -*- coding: utf-8 -*-
|
3 |
-
|
4 |
-
# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
|
5 |
-
# holder of all proprietary rights on this computer program.
|
6 |
-
# You can only use this computer program if you have closed
|
7 |
-
# a license agreement with MPG or you get the right to use the computer
|
8 |
-
# program from someone who is authorized to grant you that right.
|
9 |
-
# Any use of the computer program without a valid license is prohibited and
|
10 |
-
# liable to prosecution.
|
11 |
-
#
|
12 |
-
# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
|
13 |
-
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
|
14 |
-
# for Intelligent Systems. All rights reserved.
|
15 |
-
#
|
16 |
-
# Contact: [email protected]
|
17 |
-
|
18 |
-
from lib.dataset.mesh_util import SMPLX
|
19 |
-
from lib.common.render_utils import face_vertices
|
20 |
-
import numpy as np
|
21 |
-
import lib.smplx as smplx
|
22 |
-
import trimesh
|
23 |
-
import torch
|
24 |
-
import torch.nn.functional as F
|
25 |
-
|
26 |
-
model_init_params = dict(
|
27 |
-
gender='male',
|
28 |
-
model_type='smplx',
|
29 |
-
model_path=SMPLX().model_dir,
|
30 |
-
create_global_orient=False,
|
31 |
-
create_body_pose=False,
|
32 |
-
create_betas=False,
|
33 |
-
create_left_hand_pose=False,
|
34 |
-
create_right_hand_pose=False,
|
35 |
-
create_expression=False,
|
36 |
-
create_jaw_pose=False,
|
37 |
-
create_leye_pose=False,
|
38 |
-
create_reye_pose=False,
|
39 |
-
create_transl=False,
|
40 |
-
num_pca_comps=12)
|
41 |
-
|
42 |
-
|
43 |
-
def get_smpl_model(model_type, gender): return smplx.create(
|
44 |
-
**model_init_params)
|
45 |
-
|
46 |
-
|
47 |
-
def normalization(data):
|
48 |
-
_range = np.max(data) - np.min(data)
|
49 |
-
return ((data - np.min(data)) / _range)
|
50 |
-
|
51 |
-
|
52 |
-
def sigmoid(x):
|
53 |
-
z = 1 / (1 + np.exp(-x))
|
54 |
-
return z
|
55 |
-
|
56 |
-
|
57 |
-
def load_fit_body(fitted_path, scale, smpl_type='smplx', smpl_gender='neutral', noise_dict=None):
|
58 |
-
|
59 |
-
param = np.load(fitted_path, allow_pickle=True)
|
60 |
-
for key in param.keys():
|
61 |
-
param[key] = torch.as_tensor(param[key])
|
62 |
-
|
63 |
-
smpl_model = get_smpl_model(smpl_type, smpl_gender)
|
64 |
-
model_forward_params = dict(betas=param['betas'],
|
65 |
-
global_orient=param['global_orient'],
|
66 |
-
body_pose=param['body_pose'],
|
67 |
-
left_hand_pose=param['left_hand_pose'],
|
68 |
-
right_hand_pose=param['right_hand_pose'],
|
69 |
-
jaw_pose=param['jaw_pose'],
|
70 |
-
leye_pose=param['leye_pose'],
|
71 |
-
reye_pose=param['reye_pose'],
|
72 |
-
expression=param['expression'],
|
73 |
-
return_verts=True)
|
74 |
-
|
75 |
-
if noise_dict is not None:
|
76 |
-
model_forward_params.update(noise_dict)
|
77 |
-
|
78 |
-
smpl_out = smpl_model(**model_forward_params)
|
79 |
-
|
80 |
-
smpl_verts = (
|
81 |
-
(smpl_out.vertices[0] * param['scale'] + param['translation']) * scale).detach()
|
82 |
-
smpl_joints = (
|
83 |
-
(smpl_out.joints[0] * param['scale'] + param['translation']) * scale).detach()
|
84 |
-
smpl_mesh = trimesh.Trimesh(smpl_verts,
|
85 |
-
smpl_model.faces,
|
86 |
-
process=False, maintain_order=True)
|
87 |
-
|
88 |
-
return smpl_mesh, smpl_joints
|
89 |
-
|
90 |
-
|
91 |
-
def load_ori_fit_body(fitted_path, smpl_type='smplx', smpl_gender='neutral'):
|
92 |
-
|
93 |
-
param = np.load(fitted_path, allow_pickle=True)
|
94 |
-
for key in param.keys():
|
95 |
-
param[key] = torch.as_tensor(param[key])
|
96 |
-
|
97 |
-
smpl_model = get_smpl_model(smpl_type, smpl_gender)
|
98 |
-
model_forward_params = dict(betas=param['betas'],
|
99 |
-
global_orient=param['global_orient'],
|
100 |
-
body_pose=param['body_pose'],
|
101 |
-
left_hand_pose=param['left_hand_pose'],
|
102 |
-
right_hand_pose=param['right_hand_pose'],
|
103 |
-
jaw_pose=param['jaw_pose'],
|
104 |
-
leye_pose=param['leye_pose'],
|
105 |
-
reye_pose=param['reye_pose'],
|
106 |
-
expression=param['expression'],
|
107 |
-
return_verts=True)
|
108 |
-
|
109 |
-
smpl_out = smpl_model(**model_forward_params)
|
110 |
-
|
111 |
-
smpl_verts = smpl_out.vertices[0].detach()
|
112 |
-
smpl_mesh = trimesh.Trimesh(smpl_verts,
|
113 |
-
smpl_model.faces,
|
114 |
-
process=False, maintain_order=True)
|
115 |
-
|
116 |
-
return smpl_mesh
|
117 |
-
|
118 |
-
|
119 |
-
def save_obj_mesh(mesh_path, verts, faces):
|
120 |
-
file = open(mesh_path, 'w')
|
121 |
-
for v in verts:
|
122 |
-
file.write('v %.4f %.4f %.4f\n' % (v[0], v[1], v[2]))
|
123 |
-
for f in faces:
|
124 |
-
f_plus = f + 1
|
125 |
-
file.write('f %d %d %d\n' % (f_plus[0], f_plus[1], f_plus[2]))
|
126 |
-
file.close()
|
127 |
-
|
128 |
-
|
129 |
-
# https://github.com/ratcave/wavefront_reader
|
130 |
-
def read_mtlfile(fname):
|
131 |
-
materials = {}
|
132 |
-
with open(fname) as f:
|
133 |
-
lines = f.read().splitlines()
|
134 |
-
|
135 |
-
for line in lines:
|
136 |
-
if line:
|
137 |
-
split_line = line.strip().split(' ', 1)
|
138 |
-
if len(split_line) < 2:
|
139 |
-
continue
|
140 |
-
|
141 |
-
prefix, data = split_line[0], split_line[1]
|
142 |
-
if 'newmtl' in prefix:
|
143 |
-
material = {}
|
144 |
-
materials[data] = material
|
145 |
-
elif materials:
|
146 |
-
if data:
|
147 |
-
split_data = data.strip().split(' ')
|
148 |
-
|
149 |
-
# assume texture maps are in the same level
|
150 |
-
# WARNING: do not include space in your filename!!
|
151 |
-
if 'map' in prefix:
|
152 |
-
material[prefix] = split_data[-1].split('\\')[-1]
|
153 |
-
elif len(split_data) > 1:
|
154 |
-
material[prefix] = tuple(float(d) for d in split_data)
|
155 |
-
else:
|
156 |
-
try:
|
157 |
-
material[prefix] = int(data)
|
158 |
-
except ValueError:
|
159 |
-
material[prefix] = float(data)
|
160 |
-
|
161 |
-
return materials
|
162 |
-
|
163 |
-
|
164 |
-
def load_obj_mesh_mtl(mesh_file):
|
165 |
-
vertex_data = []
|
166 |
-
norm_data = []
|
167 |
-
uv_data = []
|
168 |
-
|
169 |
-
face_data = []
|
170 |
-
face_norm_data = []
|
171 |
-
face_uv_data = []
|
172 |
-
|
173 |
-
# face per material
|
174 |
-
face_data_mat = {}
|
175 |
-
face_norm_data_mat = {}
|
176 |
-
face_uv_data_mat = {}
|
177 |
-
|
178 |
-
# current material name
|
179 |
-
mtl_data = None
|
180 |
-
cur_mat = None
|
181 |
-
|
182 |
-
if isinstance(mesh_file, str):
|
183 |
-
f = open(mesh_file, "r")
|
184 |
-
else:
|
185 |
-
f = mesh_file
|
186 |
-
for line in f:
|
187 |
-
if isinstance(line, bytes):
|
188 |
-
line = line.decode("utf-8")
|
189 |
-
if line.startswith('#'):
|
190 |
-
continue
|
191 |
-
values = line.split()
|
192 |
-
if not values:
|
193 |
-
continue
|
194 |
-
|
195 |
-
if values[0] == 'v':
|
196 |
-
v = list(map(float, values[1:4]))
|
197 |
-
vertex_data.append(v)
|
198 |
-
elif values[0] == 'vn':
|
199 |
-
vn = list(map(float, values[1:4]))
|
200 |
-
norm_data.append(vn)
|
201 |
-
elif values[0] == 'vt':
|
202 |
-
vt = list(map(float, values[1:3]))
|
203 |
-
uv_data.append(vt)
|
204 |
-
elif values[0] == 'mtllib':
|
205 |
-
mtl_data = read_mtlfile(
|
206 |
-
mesh_file.replace(mesh_file.split('/')[-1], values[1]))
|
207 |
-
elif values[0] == 'usemtl':
|
208 |
-
cur_mat = values[1]
|
209 |
-
elif values[0] == 'f':
|
210 |
-
# local triangle data
|
211 |
-
l_face_data = []
|
212 |
-
l_face_uv_data = []
|
213 |
-
l_face_norm_data = []
|
214 |
-
|
215 |
-
# quad mesh
|
216 |
-
if len(values) > 4:
|
217 |
-
f = list(
|
218 |
-
map(
|
219 |
-
lambda x: int(x.split('/')[0]) if int(x.split('/')[0])
|
220 |
-
< 0 else int(x.split('/')[0]) - 1, values[1:4]))
|
221 |
-
l_face_data.append(f)
|
222 |
-
f = list(
|
223 |
-
map(
|
224 |
-
lambda x: int(x.split('/')[0])
|
225 |
-
if int(x.split('/')[0]) < 0 else int(x.split('/')[0]) -
|
226 |
-
1, [values[3], values[4], values[1]]))
|
227 |
-
l_face_data.append(f)
|
228 |
-
# tri mesh
|
229 |
-
else:
|
230 |
-
f = list(
|
231 |
-
map(
|
232 |
-
lambda x: int(x.split('/')[0]) if int(x.split('/')[0])
|
233 |
-
< 0 else int(x.split('/')[0]) - 1, values[1:4]))
|
234 |
-
l_face_data.append(f)
|
235 |
-
# deal with texture
|
236 |
-
if len(values[1].split('/')) >= 2:
|
237 |
-
# quad mesh
|
238 |
-
if len(values) > 4:
|
239 |
-
f = list(
|
240 |
-
map(
|
241 |
-
lambda x: int(x.split('/')[1])
|
242 |
-
if int(x.split('/')[1]) < 0 else int(
|
243 |
-
x.split('/')[1]) - 1, values[1:4]))
|
244 |
-
l_face_uv_data.append(f)
|
245 |
-
f = list(
|
246 |
-
map(
|
247 |
-
lambda x: int(x.split('/')[1])
|
248 |
-
if int(x.split('/')[1]) < 0 else int(
|
249 |
-
x.split('/')[1]) - 1,
|
250 |
-
[values[3], values[4], values[1]]))
|
251 |
-
l_face_uv_data.append(f)
|
252 |
-
# tri mesh
|
253 |
-
elif len(values[1].split('/')[1]) != 0:
|
254 |
-
f = list(
|
255 |
-
map(
|
256 |
-
lambda x: int(x.split('/')[1])
|
257 |
-
if int(x.split('/')[1]) < 0 else int(
|
258 |
-
x.split('/')[1]) - 1, values[1:4]))
|
259 |
-
l_face_uv_data.append(f)
|
260 |
-
# deal with normal
|
261 |
-
if len(values[1].split('/')) == 3:
|
262 |
-
# quad mesh
|
263 |
-
if len(values) > 4:
|
264 |
-
f = list(
|
265 |
-
map(
|
266 |
-
lambda x: int(x.split('/')[2])
|
267 |
-
if int(x.split('/')[2]) < 0 else int(
|
268 |
-
x.split('/')[2]) - 1, values[1:4]))
|
269 |
-
l_face_norm_data.append(f)
|
270 |
-
f = list(
|
271 |
-
map(
|
272 |
-
lambda x: int(x.split('/')[2])
|
273 |
-
if int(x.split('/')[2]) < 0 else int(
|
274 |
-
x.split('/')[2]) - 1,
|
275 |
-
[values[3], values[4], values[1]]))
|
276 |
-
l_face_norm_data.append(f)
|
277 |
-
# tri mesh
|
278 |
-
elif len(values[1].split('/')[2]) != 0:
|
279 |
-
f = list(
|
280 |
-
map(
|
281 |
-
lambda x: int(x.split('/')[2])
|
282 |
-
if int(x.split('/')[2]) < 0 else int(
|
283 |
-
x.split('/')[2]) - 1, values[1:4]))
|
284 |
-
l_face_norm_data.append(f)
|
285 |
-
|
286 |
-
face_data += l_face_data
|
287 |
-
face_uv_data += l_face_uv_data
|
288 |
-
face_norm_data += l_face_norm_data
|
289 |
-
|
290 |
-
if cur_mat is not None:
|
291 |
-
if cur_mat not in face_data_mat.keys():
|
292 |
-
face_data_mat[cur_mat] = []
|
293 |
-
if cur_mat not in face_uv_data_mat.keys():
|
294 |
-
face_uv_data_mat[cur_mat] = []
|
295 |
-
if cur_mat not in face_norm_data_mat.keys():
|
296 |
-
face_norm_data_mat[cur_mat] = []
|
297 |
-
face_data_mat[cur_mat] += l_face_data
|
298 |
-
face_uv_data_mat[cur_mat] += l_face_uv_data
|
299 |
-
face_norm_data_mat[cur_mat] += l_face_norm_data
|
300 |
-
|
301 |
-
vertices = np.array(vertex_data)
|
302 |
-
faces = np.array(face_data)
|
303 |
-
|
304 |
-
norms = np.array(norm_data)
|
305 |
-
norms = normalize_v3(norms)
|
306 |
-
face_normals = np.array(face_norm_data)
|
307 |
-
|
308 |
-
uvs = np.array(uv_data)
|
309 |
-
face_uvs = np.array(face_uv_data)
|
310 |
-
|
311 |
-
out_tuple = (vertices, faces, norms, face_normals, uvs, face_uvs)
|
312 |
-
|
313 |
-
if cur_mat is not None and mtl_data is not None:
|
314 |
-
for key in face_data_mat:
|
315 |
-
face_data_mat[key] = np.array(face_data_mat[key])
|
316 |
-
face_uv_data_mat[key] = np.array(face_uv_data_mat[key])
|
317 |
-
face_norm_data_mat[key] = np.array(face_norm_data_mat[key])
|
318 |
-
|
319 |
-
out_tuple += (face_data_mat, face_norm_data_mat, face_uv_data_mat,
|
320 |
-
mtl_data)
|
321 |
-
|
322 |
-
return out_tuple
|
323 |
-
|
324 |
-
|
325 |
-
def load_scan(mesh_file, with_normal=False, with_texture=False):
|
326 |
-
vertex_data = []
|
327 |
-
norm_data = []
|
328 |
-
uv_data = []
|
329 |
-
|
330 |
-
face_data = []
|
331 |
-
face_norm_data = []
|
332 |
-
face_uv_data = []
|
333 |
-
|
334 |
-
if isinstance(mesh_file, str):
|
335 |
-
f = open(mesh_file, "r")
|
336 |
-
else:
|
337 |
-
f = mesh_file
|
338 |
-
for line in f:
|
339 |
-
if isinstance(line, bytes):
|
340 |
-
line = line.decode("utf-8")
|
341 |
-
if line.startswith('#'):
|
342 |
-
continue
|
343 |
-
values = line.split()
|
344 |
-
if not values:
|
345 |
-
continue
|
346 |
-
|
347 |
-
if values[0] == 'v':
|
348 |
-
v = list(map(float, values[1:4]))
|
349 |
-
vertex_data.append(v)
|
350 |
-
elif values[0] == 'vn':
|
351 |
-
vn = list(map(float, values[1:4]))
|
352 |
-
norm_data.append(vn)
|
353 |
-
elif values[0] == 'vt':
|
354 |
-
vt = list(map(float, values[1:3]))
|
355 |
-
uv_data.append(vt)
|
356 |
-
|
357 |
-
elif values[0] == 'f':
|
358 |
-
# quad mesh
|
359 |
-
if len(values) > 4:
|
360 |
-
f = list(map(lambda x: int(x.split('/')[0]), values[1:4]))
|
361 |
-
face_data.append(f)
|
362 |
-
f = list(
|
363 |
-
map(lambda x: int(x.split('/')[0]),
|
364 |
-
[values[3], values[4], values[1]]))
|
365 |
-
face_data.append(f)
|
366 |
-
# tri mesh
|
367 |
-
else:
|
368 |
-
f = list(map(lambda x: int(x.split('/')[0]), values[1:4]))
|
369 |
-
face_data.append(f)
|
370 |
-
|
371 |
-
# deal with texture
|
372 |
-
if len(values[1].split('/')) >= 2:
|
373 |
-
# quad mesh
|
374 |
-
if len(values) > 4:
|
375 |
-
f = list(map(lambda x: int(x.split('/')[1]), values[1:4]))
|
376 |
-
face_uv_data.append(f)
|
377 |
-
f = list(
|
378 |
-
map(lambda x: int(x.split('/')[1]),
|
379 |
-
[values[3], values[4], values[1]]))
|
380 |
-
face_uv_data.append(f)
|
381 |
-
# tri mesh
|
382 |
-
elif len(values[1].split('/')[1]) != 0:
|
383 |
-
f = list(map(lambda x: int(x.split('/')[1]), values[1:4]))
|
384 |
-
face_uv_data.append(f)
|
385 |
-
# deal with normal
|
386 |
-
if len(values[1].split('/')) == 3:
|
387 |
-
# quad mesh
|
388 |
-
if len(values) > 4:
|
389 |
-
f = list(map(lambda x: int(x.split('/')[2]), values[1:4]))
|
390 |
-
face_norm_data.append(f)
|
391 |
-
f = list(
|
392 |
-
map(lambda x: int(x.split('/')[2]),
|
393 |
-
[values[3], values[4], values[1]]))
|
394 |
-
face_norm_data.append(f)
|
395 |
-
# tri mesh
|
396 |
-
elif len(values[1].split('/')[2]) != 0:
|
397 |
-
f = list(map(lambda x: int(x.split('/')[2]), values[1:4]))
|
398 |
-
face_norm_data.append(f)
|
399 |
-
|
400 |
-
vertices = np.array(vertex_data)
|
401 |
-
faces = np.array(face_data) - 1
|
402 |
-
|
403 |
-
if with_texture and with_normal:
|
404 |
-
uvs = np.array(uv_data)
|
405 |
-
face_uvs = np.array(face_uv_data) - 1
|
406 |
-
norms = np.array(norm_data)
|
407 |
-
if norms.shape[0] == 0:
|
408 |
-
norms = compute_normal(vertices, faces)
|
409 |
-
face_normals = faces
|
410 |
-
else:
|
411 |
-
norms = normalize_v3(norms)
|
412 |
-
face_normals = np.array(face_norm_data) - 1
|
413 |
-
return vertices, faces, norms, face_normals, uvs, face_uvs
|
414 |
-
|
415 |
-
if with_texture:
|
416 |
-
uvs = np.array(uv_data)
|
417 |
-
face_uvs = np.array(face_uv_data) - 1
|
418 |
-
return vertices, faces, uvs, face_uvs
|
419 |
-
|
420 |
-
if with_normal:
|
421 |
-
norms = np.array(norm_data)
|
422 |
-
norms = normalize_v3(norms)
|
423 |
-
face_normals = np.array(face_norm_data) - 1
|
424 |
-
return vertices, faces, norms, face_normals
|
425 |
-
|
426 |
-
return vertices, faces
|
427 |
-
|
428 |
-
|
429 |
-
def normalize_v3(arr):
|
430 |
-
''' Normalize a numpy array of 3 component vectors shape=(n,3) '''
|
431 |
-
lens = np.sqrt(arr[:, 0]**2 + arr[:, 1]**2 + arr[:, 2]**2)
|
432 |
-
eps = 0.00000001
|
433 |
-
lens[lens < eps] = eps
|
434 |
-
arr[:, 0] /= lens
|
435 |
-
arr[:, 1] /= lens
|
436 |
-
arr[:, 2] /= lens
|
437 |
-
return arr
|
438 |
-
|
439 |
-
|
440 |
-
def compute_normal(vertices, faces):
|
441 |
-
# Create a zeroed array with the same type and shape as our vertices i.e., per vertex normal
|
442 |
-
norm = np.zeros(vertices.shape, dtype=vertices.dtype)
|
443 |
-
# Create an indexed view into the vertex array using the array of three indices for triangles
|
444 |
-
tris = vertices[faces]
|
445 |
-
# Calculate the normal for all the triangles, by taking the cross product of the vectors v1-v0, and v2-v0 in each triangle
|
446 |
-
n = np.cross(tris[::, 1] - tris[::, 0], tris[::, 2] - tris[::, 0])
|
447 |
-
# n is now an array of normals per triangle. The length of each normal is dependent the vertices,
|
448 |
-
# we need to normalize these, so that our next step weights each normal equally.
|
449 |
-
normalize_v3(n)
|
450 |
-
# now we have a normalized array of normals, one per triangle, i.e., per triangle normals.
|
451 |
-
# But instead of one per triangle (i.e., flat shading), we add to each vertex in that triangle,
|
452 |
-
# the triangles' normal. Multiple triangles would then contribute to every vertex, so we need to normalize again afterwards.
|
453 |
-
# The cool part, we can actually add the normals through an indexed view of our (zeroed) per vertex normal array
|
454 |
-
norm[faces[:, 0]] += n
|
455 |
-
norm[faces[:, 1]] += n
|
456 |
-
norm[faces[:, 2]] += n
|
457 |
-
normalize_v3(norm)
|
458 |
-
|
459 |
-
return norm
|
460 |
-
|
461 |
-
|
462 |
-
def compute_normal_batch(vertices, faces):
|
463 |
-
|
464 |
-
bs, nv = vertices.shape[:2]
|
465 |
-
bs, nf = faces.shape[:2]
|
466 |
-
|
467 |
-
vert_norm = torch.zeros(bs * nv, 3).type_as(vertices)
|
468 |
-
tris = face_vertices(vertices, faces)
|
469 |
-
face_norm = F.normalize(torch.cross(tris[:, :, 1] - tris[:, :, 0],
|
470 |
-
tris[:, :, 2] - tris[:, :, 0]),
|
471 |
-
dim=-1)
|
472 |
-
|
473 |
-
faces = (faces +
|
474 |
-
(torch.arange(bs).type_as(faces) * nv)[:, None, None]).view(
|
475 |
-
-1, 3)
|
476 |
-
|
477 |
-
vert_norm[faces[:, 0]] += face_norm.view(-1, 3)
|
478 |
-
vert_norm[faces[:, 1]] += face_norm.view(-1, 3)
|
479 |
-
vert_norm[faces[:, 2]] += face_norm.view(-1, 3)
|
480 |
-
|
481 |
-
vert_norm = F.normalize(vert_norm, dim=-1).view(bs, nv, 3)
|
482 |
-
|
483 |
-
return vert_norm
|
484 |
-
|
485 |
-
|
486 |
-
# compute tangent and bitangent
|
487 |
-
def compute_tangent(vertices, faces, normals, uvs, faceuvs):
|
488 |
-
# NOTE: this could be numerically unstable around [0,0,1]
|
489 |
-
# but other current solutions are pretty freaky somehow
|
490 |
-
c1 = np.cross(normals, np.array([0, 1, 0.0]))
|
491 |
-
tan = c1
|
492 |
-
normalize_v3(tan)
|
493 |
-
btan = np.cross(normals, tan)
|
494 |
-
|
495 |
-
# NOTE: traditional version is below
|
496 |
-
|
497 |
-
# pts_tris = vertices[faces]
|
498 |
-
# uv_tris = uvs[faceuvs]
|
499 |
-
|
500 |
-
# W = np.stack([pts_tris[::, 1] - pts_tris[::, 0], pts_tris[::, 2] - pts_tris[::, 0]],2)
|
501 |
-
# UV = np.stack([uv_tris[::, 1] - uv_tris[::, 0], uv_tris[::, 2] - uv_tris[::, 0]], 1)
|
502 |
-
|
503 |
-
# for i in range(W.shape[0]):
|
504 |
-
# W[i,::] = W[i,::].dot(np.linalg.inv(UV[i,::]))
|
505 |
-
|
506 |
-
# tan = np.zeros(vertices.shape, dtype=vertices.dtype)
|
507 |
-
# tan[faces[:,0]] += W[:,:,0]
|
508 |
-
# tan[faces[:,1]] += W[:,:,0]
|
509 |
-
# tan[faces[:,2]] += W[:,:,0]
|
510 |
-
|
511 |
-
# btan = np.zeros(vertices.shape, dtype=vertices.dtype)
|
512 |
-
# btan[faces[:,0]] += W[:,:,1]
|
513 |
-
# btan[faces[:,1]] += W[:,:,1]
|
514 |
-
# btan[faces[:,2]] += W[:,:,1]
|
515 |
-
|
516 |
-
# normalize_v3(tan)
|
517 |
-
|
518 |
-
# ndott = np.sum(normals*tan, 1, keepdims=True)
|
519 |
-
# tan = tan - ndott * normals
|
520 |
-
|
521 |
-
# normalize_v3(btan)
|
522 |
-
# normalize_v3(tan)
|
523 |
-
|
524 |
-
# tan[np.sum(np.cross(normals, tan) * btan, 1) < 0,:] *= -1.0
|
525 |
-
|
526 |
-
return tan, btan
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer /opengl_util.py
DELETED
@@ -1,369 +0,0 @@
|
|
1 |
-
|
2 |
-
# -*- coding: utf-8 -*-
|
3 |
-
|
4 |
-
# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
|
5 |
-
# holder of all proprietary rights on this computer program.
|
6 |
-
# You can only use this computer program if you have closed
|
7 |
-
# a license agreement with MPG or you get the right to use the computer
|
8 |
-
# program from someone who is authorized to grant you that right.
|
9 |
-
# Any use of the computer program without a valid license is prohibited and
|
10 |
-
# liable to prosecution.
|
11 |
-
#
|
12 |
-
# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
|
13 |
-
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
|
14 |
-
# for Intelligent Systems. All rights reserved.
|
15 |
-
#
|
16 |
-
# Contact: [email protected]
|
17 |
-
|
18 |
-
import os
|
19 |
-
|
20 |
-
from lib.renderer.mesh import load_scan, compute_tangent
|
21 |
-
from lib.renderer.camera import Camera
|
22 |
-
import cv2
|
23 |
-
import math
|
24 |
-
import random
|
25 |
-
import numpy as np
|
26 |
-
|
27 |
-
|
28 |
-
def render_result(rndr, shader_id, path, mask=False):
|
29 |
-
|
30 |
-
cam_render = rndr.get_color(shader_id)
|
31 |
-
cam_render = cv2.cvtColor(cam_render, cv2.COLOR_RGBA2BGRA)
|
32 |
-
|
33 |
-
os.makedirs(os.path.dirname(path), exist_ok=True)
|
34 |
-
if shader_id != 2:
|
35 |
-
cv2.imwrite(path, np.uint8(255.0 * cam_render))
|
36 |
-
else:
|
37 |
-
cam_render[:, :, -1] -= 0.5
|
38 |
-
cam_render[:, :, -1] *= 2.0
|
39 |
-
if not mask:
|
40 |
-
cv2.imwrite(path, np.uint8(255.0 / 2.0 * (cam_render + 1.0)))
|
41 |
-
else:
|
42 |
-
cv2.imwrite(path, np.uint8(-1.0 * cam_render[:, :, [3]]))
|
43 |
-
|
44 |
-
|
45 |
-
def make_rotate(rx, ry, rz):
|
46 |
-
sinX = np.sin(rx)
|
47 |
-
sinY = np.sin(ry)
|
48 |
-
sinZ = np.sin(rz)
|
49 |
-
|
50 |
-
cosX = np.cos(rx)
|
51 |
-
cosY = np.cos(ry)
|
52 |
-
cosZ = np.cos(rz)
|
53 |
-
|
54 |
-
Rx = np.zeros((3, 3))
|
55 |
-
Rx[0, 0] = 1.0
|
56 |
-
Rx[1, 1] = cosX
|
57 |
-
Rx[1, 2] = -sinX
|
58 |
-
Rx[2, 1] = sinX
|
59 |
-
Rx[2, 2] = cosX
|
60 |
-
|
61 |
-
Ry = np.zeros((3, 3))
|
62 |
-
Ry[0, 0] = cosY
|
63 |
-
Ry[0, 2] = sinY
|
64 |
-
Ry[1, 1] = 1.0
|
65 |
-
Ry[2, 0] = -sinY
|
66 |
-
Ry[2, 2] = cosY
|
67 |
-
|
68 |
-
Rz = np.zeros((3, 3))
|
69 |
-
Rz[0, 0] = cosZ
|
70 |
-
Rz[0, 1] = -sinZ
|
71 |
-
Rz[1, 0] = sinZ
|
72 |
-
Rz[1, 1] = cosZ
|
73 |
-
Rz[2, 2] = 1.0
|
74 |
-
|
75 |
-
R = np.matmul(np.matmul(Rz, Ry), Rx)
|
76 |
-
return R
|
77 |
-
|
78 |
-
|
79 |
-
def rotateSH(SH, R):
|
80 |
-
SHn = SH
|
81 |
-
|
82 |
-
# 1st order
|
83 |
-
SHn[1] = R[1, 1] * SH[1] - R[1, 2] * SH[2] + R[1, 0] * SH[3]
|
84 |
-
SHn[2] = -R[2, 1] * SH[1] + R[2, 2] * SH[2] - R[2, 0] * SH[3]
|
85 |
-
SHn[3] = R[0, 1] * SH[1] - R[0, 2] * SH[2] + R[0, 0] * SH[3]
|
86 |
-
|
87 |
-
# 2nd order
|
88 |
-
SHn[4:, 0] = rotateBand2(SH[4:, 0], R)
|
89 |
-
SHn[4:, 1] = rotateBand2(SH[4:, 1], R)
|
90 |
-
SHn[4:, 2] = rotateBand2(SH[4:, 2], R)
|
91 |
-
|
92 |
-
return SHn
|
93 |
-
|
94 |
-
|
95 |
-
def rotateBand2(x, R):
|
96 |
-
s_c3 = 0.94617469575
|
97 |
-
s_c4 = -0.31539156525
|
98 |
-
s_c5 = 0.54627421529
|
99 |
-
|
100 |
-
s_c_scale = 1.0 / 0.91529123286551084
|
101 |
-
s_c_scale_inv = 0.91529123286551084
|
102 |
-
|
103 |
-
s_rc2 = 1.5853309190550713 * s_c_scale
|
104 |
-
s_c4_div_c3 = s_c4 / s_c3
|
105 |
-
s_c4_div_c3_x2 = (s_c4 / s_c3) * 2.0
|
106 |
-
|
107 |
-
s_scale_dst2 = s_c3 * s_c_scale_inv
|
108 |
-
s_scale_dst4 = s_c5 * s_c_scale_inv
|
109 |
-
|
110 |
-
sh0 = x[3] + x[4] + x[4] - x[1]
|
111 |
-
sh1 = x[0] + s_rc2 * x[2] + x[3] + x[4]
|
112 |
-
sh2 = x[0]
|
113 |
-
sh3 = -x[3]
|
114 |
-
sh4 = -x[1]
|
115 |
-
|
116 |
-
r2x = R[0][0] + R[0][1]
|
117 |
-
r2y = R[1][0] + R[1][1]
|
118 |
-
r2z = R[2][0] + R[2][1]
|
119 |
-
|
120 |
-
r3x = R[0][0] + R[0][2]
|
121 |
-
r3y = R[1][0] + R[1][2]
|
122 |
-
r3z = R[2][0] + R[2][2]
|
123 |
-
|
124 |
-
r4x = R[0][1] + R[0][2]
|
125 |
-
r4y = R[1][1] + R[1][2]
|
126 |
-
r4z = R[2][1] + R[2][2]
|
127 |
-
|
128 |
-
sh0_x = sh0 * R[0][0]
|
129 |
-
sh0_y = sh0 * R[1][0]
|
130 |
-
d0 = sh0_x * R[1][0]
|
131 |
-
d1 = sh0_y * R[2][0]
|
132 |
-
d2 = sh0 * (R[2][0] * R[2][0] + s_c4_div_c3)
|
133 |
-
d3 = sh0_x * R[2][0]
|
134 |
-
d4 = sh0_x * R[0][0] - sh0_y * R[1][0]
|
135 |
-
|
136 |
-
sh1_x = sh1 * R[0][2]
|
137 |
-
sh1_y = sh1 * R[1][2]
|
138 |
-
d0 += sh1_x * R[1][2]
|
139 |
-
d1 += sh1_y * R[2][2]
|
140 |
-
d2 += sh1 * (R[2][2] * R[2][2] + s_c4_div_c3)
|
141 |
-
d3 += sh1_x * R[2][2]
|
142 |
-
d4 += sh1_x * R[0][2] - sh1_y * R[1][2]
|
143 |
-
|
144 |
-
sh2_x = sh2 * r2x
|
145 |
-
sh2_y = sh2 * r2y
|
146 |
-
d0 += sh2_x * r2y
|
147 |
-
d1 += sh2_y * r2z
|
148 |
-
d2 += sh2 * (r2z * r2z + s_c4_div_c3_x2)
|
149 |
-
d3 += sh2_x * r2z
|
150 |
-
d4 += sh2_x * r2x - sh2_y * r2y
|
151 |
-
|
152 |
-
sh3_x = sh3 * r3x
|
153 |
-
sh3_y = sh3 * r3y
|
154 |
-
d0 += sh3_x * r3y
|
155 |
-
d1 += sh3_y * r3z
|
156 |
-
d2 += sh3 * (r3z * r3z + s_c4_div_c3_x2)
|
157 |
-
d3 += sh3_x * r3z
|
158 |
-
d4 += sh3_x * r3x - sh3_y * r3y
|
159 |
-
|
160 |
-
sh4_x = sh4 * r4x
|
161 |
-
sh4_y = sh4 * r4y
|
162 |
-
d0 += sh4_x * r4y
|
163 |
-
d1 += sh4_y * r4z
|
164 |
-
d2 += sh4 * (r4z * r4z + s_c4_div_c3_x2)
|
165 |
-
d3 += sh4_x * r4z
|
166 |
-
d4 += sh4_x * r4x - sh4_y * r4y
|
167 |
-
|
168 |
-
dst = x
|
169 |
-
dst[0] = d0
|
170 |
-
dst[1] = -d1
|
171 |
-
dst[2] = d2 * s_scale_dst2
|
172 |
-
dst[3] = -d3
|
173 |
-
dst[4] = d4 * s_scale_dst4
|
174 |
-
|
175 |
-
return dst
|
176 |
-
|
177 |
-
|
178 |
-
def load_calib(param, render_size=512):
|
179 |
-
# pixel unit / world unit
|
180 |
-
ortho_ratio = param['ortho_ratio']
|
181 |
-
# world unit / model unit
|
182 |
-
scale = param['scale']
|
183 |
-
# camera center world coordinate
|
184 |
-
center = param['center']
|
185 |
-
# model rotation
|
186 |
-
R = param['R']
|
187 |
-
|
188 |
-
translate = -np.matmul(R, center).reshape(3, 1)
|
189 |
-
extrinsic = np.concatenate([R, translate], axis=1)
|
190 |
-
extrinsic = np.concatenate(
|
191 |
-
[extrinsic, np.array([0, 0, 0, 1]).reshape(1, 4)], 0)
|
192 |
-
# Match camera space to image pixel space
|
193 |
-
scale_intrinsic = np.identity(4)
|
194 |
-
scale_intrinsic[0, 0] = scale / ortho_ratio
|
195 |
-
scale_intrinsic[1, 1] = -scale / ortho_ratio
|
196 |
-
scale_intrinsic[2, 2] = scale / ortho_ratio
|
197 |
-
# Match image pixel space to image uv space
|
198 |
-
uv_intrinsic = np.identity(4)
|
199 |
-
uv_intrinsic[0, 0] = 1.0 / float(render_size // 2)
|
200 |
-
uv_intrinsic[1, 1] = 1.0 / float(render_size // 2)
|
201 |
-
uv_intrinsic[2, 2] = 1.0 / float(render_size // 2)
|
202 |
-
|
203 |
-
intrinsic = np.matmul(uv_intrinsic, scale_intrinsic)
|
204 |
-
calib = np.concatenate([extrinsic, intrinsic], axis=0)
|
205 |
-
return calib
|
206 |
-
|
207 |
-
|
208 |
-
def render_prt_ortho(out_path,
|
209 |
-
folder_name,
|
210 |
-
subject_name,
|
211 |
-
shs,
|
212 |
-
rndr,
|
213 |
-
rndr_uv,
|
214 |
-
im_size,
|
215 |
-
angl_step=4,
|
216 |
-
n_light=1,
|
217 |
-
pitch=[0]):
|
218 |
-
cam = Camera(width=im_size, height=im_size)
|
219 |
-
cam.ortho_ratio = 0.4 * (512 / im_size)
|
220 |
-
cam.near = -100
|
221 |
-
cam.far = 100
|
222 |
-
cam.sanity_check()
|
223 |
-
|
224 |
-
# set path for obj, prt
|
225 |
-
mesh_file = os.path.join(folder_name, subject_name + '_100k.obj')
|
226 |
-
if not os.path.exists(mesh_file):
|
227 |
-
print('ERROR: obj file does not exist!!', mesh_file)
|
228 |
-
return
|
229 |
-
prt_file = os.path.join(folder_name, 'bounce', 'bounce0.txt')
|
230 |
-
if not os.path.exists(prt_file):
|
231 |
-
print('ERROR: prt file does not exist!!!', prt_file)
|
232 |
-
return
|
233 |
-
face_prt_file = os.path.join(folder_name, 'bounce', 'face.npy')
|
234 |
-
if not os.path.exists(face_prt_file):
|
235 |
-
print('ERROR: face prt file does not exist!!!', prt_file)
|
236 |
-
return
|
237 |
-
text_file = os.path.join(folder_name, 'tex', subject_name + '_dif_2k.jpg')
|
238 |
-
if not os.path.exists(text_file):
|
239 |
-
print('ERROR: dif file does not exist!!', text_file)
|
240 |
-
return
|
241 |
-
|
242 |
-
texture_image = cv2.imread(text_file)
|
243 |
-
texture_image = cv2.cvtColor(texture_image, cv2.COLOR_BGR2RGB)
|
244 |
-
|
245 |
-
vertices, faces, normals, faces_normals, textures, face_textures = load_scan(
|
246 |
-
mesh_file, with_normal=True, with_texture=True)
|
247 |
-
vmin = vertices.min(0)
|
248 |
-
vmax = vertices.max(0)
|
249 |
-
up_axis = 1 if (vmax - vmin).argmax() == 1 else 2
|
250 |
-
|
251 |
-
vmed = np.median(vertices, 0)
|
252 |
-
vmed[up_axis] = 0.5 * (vmax[up_axis] + vmin[up_axis])
|
253 |
-
y_scale = 180 / (vmax[up_axis] - vmin[up_axis])
|
254 |
-
|
255 |
-
rndr.set_norm_mat(y_scale, vmed)
|
256 |
-
rndr_uv.set_norm_mat(y_scale, vmed)
|
257 |
-
|
258 |
-
tan, bitan = compute_tangent(vertices, faces, normals, textures,
|
259 |
-
face_textures)
|
260 |
-
prt = np.loadtxt(prt_file)
|
261 |
-
face_prt = np.load(face_prt_file)
|
262 |
-
rndr.set_mesh(vertices, faces, normals, faces_normals, textures,
|
263 |
-
face_textures, prt, face_prt, tan, bitan)
|
264 |
-
rndr.set_albedo(texture_image)
|
265 |
-
|
266 |
-
rndr_uv.set_mesh(vertices, faces, normals, faces_normals, textures,
|
267 |
-
face_textures, prt, face_prt, tan, bitan)
|
268 |
-
rndr_uv.set_albedo(texture_image)
|
269 |
-
|
270 |
-
os.makedirs(os.path.join(out_path, 'GEO', 'OBJ', subject_name),
|
271 |
-
exist_ok=True)
|
272 |
-
os.makedirs(os.path.join(out_path, 'PARAM', subject_name), exist_ok=True)
|
273 |
-
os.makedirs(os.path.join(out_path, 'RENDER', subject_name), exist_ok=True)
|
274 |
-
os.makedirs(os.path.join(out_path, 'MASK', subject_name), exist_ok=True)
|
275 |
-
os.makedirs(os.path.join(out_path, 'UV_RENDER', subject_name),
|
276 |
-
exist_ok=True)
|
277 |
-
os.makedirs(os.path.join(out_path, 'UV_MASK', subject_name), exist_ok=True)
|
278 |
-
os.makedirs(os.path.join(out_path, 'UV_POS', subject_name), exist_ok=True)
|
279 |
-
os.makedirs(os.path.join(out_path, 'UV_NORMAL', subject_name),
|
280 |
-
exist_ok=True)
|
281 |
-
|
282 |
-
if not os.path.exists(os.path.join(out_path, 'val.txt')):
|
283 |
-
f = open(os.path.join(out_path, 'val.txt'), 'w')
|
284 |
-
f.close()
|
285 |
-
|
286 |
-
# copy obj file
|
287 |
-
cmd = 'cp %s %s' % (mesh_file,
|
288 |
-
os.path.join(out_path, 'GEO', 'OBJ', subject_name))
|
289 |
-
print(cmd)
|
290 |
-
os.system(cmd)
|
291 |
-
|
292 |
-
for p in pitch:
|
293 |
-
for y in tqdm(range(0, 360, angl_step)):
|
294 |
-
R = np.matmul(make_rotate(math.radians(p), 0, 0),
|
295 |
-
make_rotate(0, math.radians(y), 0))
|
296 |
-
if up_axis == 2:
|
297 |
-
R = np.matmul(R, make_rotate(math.radians(90), 0, 0))
|
298 |
-
|
299 |
-
rndr.rot_matrix = R
|
300 |
-
rndr_uv.rot_matrix = R
|
301 |
-
rndr.set_camera(cam)
|
302 |
-
rndr_uv.set_camera(cam)
|
303 |
-
|
304 |
-
for j in range(n_light):
|
305 |
-
sh_id = random.randint(0, shs.shape[0] - 1)
|
306 |
-
sh = shs[sh_id]
|
307 |
-
sh_angle = 0.2 * np.pi * (random.random() - 0.5)
|
308 |
-
sh = rotateSH(sh, make_rotate(0, sh_angle, 0).T)
|
309 |
-
|
310 |
-
dic = {
|
311 |
-
'sh': sh,
|
312 |
-
'ortho_ratio': cam.ortho_ratio,
|
313 |
-
'scale': y_scale,
|
314 |
-
'center': vmed,
|
315 |
-
'R': R
|
316 |
-
}
|
317 |
-
|
318 |
-
rndr.set_sh(sh)
|
319 |
-
rndr.analytic = False
|
320 |
-
rndr.use_inverse_depth = False
|
321 |
-
rndr.display()
|
322 |
-
|
323 |
-
out_all_f = rndr.get_color(0)
|
324 |
-
out_mask = out_all_f[:, :, 3]
|
325 |
-
out_all_f = cv2.cvtColor(out_all_f, cv2.COLOR_RGBA2BGR)
|
326 |
-
|
327 |
-
np.save(
|
328 |
-
os.path.join(out_path, 'PARAM', subject_name,
|
329 |
-
'%d_%d_%02d.npy' % (y, p, j)), dic)
|
330 |
-
cv2.imwrite(
|
331 |
-
os.path.join(out_path, 'RENDER', subject_name,
|
332 |
-
'%d_%d_%02d.jpg' % (y, p, j)),
|
333 |
-
255.0 * out_all_f)
|
334 |
-
cv2.imwrite(
|
335 |
-
os.path.join(out_path, 'MASK', subject_name,
|
336 |
-
'%d_%d_%02d.png' % (y, p, j)),
|
337 |
-
255.0 * out_mask)
|
338 |
-
|
339 |
-
rndr_uv.set_sh(sh)
|
340 |
-
rndr_uv.analytic = False
|
341 |
-
rndr_uv.use_inverse_depth = False
|
342 |
-
rndr_uv.display()
|
343 |
-
|
344 |
-
uv_color = rndr_uv.get_color(0)
|
345 |
-
uv_color = cv2.cvtColor(uv_color, cv2.COLOR_RGBA2BGR)
|
346 |
-
cv2.imwrite(
|
347 |
-
os.path.join(out_path, 'UV_RENDER', subject_name,
|
348 |
-
'%d_%d_%02d.jpg' % (y, p, j)),
|
349 |
-
255.0 * uv_color)
|
350 |
-
|
351 |
-
if y == 0 and j == 0 and p == pitch[0]:
|
352 |
-
uv_pos = rndr_uv.get_color(1)
|
353 |
-
uv_mask = uv_pos[:, :, 3]
|
354 |
-
cv2.imwrite(
|
355 |
-
os.path.join(out_path, 'UV_MASK', subject_name,
|
356 |
-
'00.png'), 255.0 * uv_mask)
|
357 |
-
|
358 |
-
data = {
|
359 |
-
'default': uv_pos[:, :, :3]
|
360 |
-
} # default is a reserved name
|
361 |
-
pyexr.write(
|
362 |
-
os.path.join(out_path, 'UV_POS', subject_name,
|
363 |
-
'00.exr'), data)
|
364 |
-
|
365 |
-
uv_nml = rndr_uv.get_color(2)
|
366 |
-
uv_nml = cv2.cvtColor(uv_nml, cv2.COLOR_RGBA2BGR)
|
367 |
-
cv2.imwrite(
|
368 |
-
os.path.join(out_path, 'UV_NORMAL', subject_name,
|
369 |
-
'00.png'), 255.0 * uv_nml)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib / renderer /prt_util.py
DELETED
@@ -1,199 +0,0 @@
|
|
1 |
-
|
2 |
-
# -*- coding: utf-8 -*-
|
3 |
-
|
4 |
-
# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
|
5 |
-
# holder of all proprietary rights on this computer program.
|
6 |
-
# You can only use this computer program if you have closed
|
7 |
-
# a license agreement with MPG or you get the right to use the computer
|
8 |
-
# program from someone who is authorized to grant you that right.
|
9 |
-
# Any use of the computer program without a valid license is prohibited and
|
10 |
-
# liable to prosecution.
|
11 |
-
#
|
12 |
-
# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
|
13 |
-
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
|
14 |
-
# for Intelligent Systems. All rights reserved.
|
15 |
-
#
|
16 |
-
# Contact: [email protected]
|
17 |
-
|
18 |
-
import os
|
19 |
-
import trimesh
|
20 |
-
import numpy as np
|
21 |
-
import math
|
22 |
-
from scipy.special import sph_harm
|
23 |
-
import argparse
|
24 |
-
from tqdm import tqdm
|
25 |
-
from trimesh.util import bounds_tree
|
26 |
-
|
27 |
-
|
28 |
-
def factratio(N, D):
|
29 |
-
if N >= D:
|
30 |
-
prod = 1.0
|
31 |
-
for i in range(D + 1, N + 1):
|
32 |
-
prod *= i
|
33 |
-
return prod
|
34 |
-
else:
|
35 |
-
prod = 1.0
|
36 |
-
for i in range(N + 1, D + 1):
|
37 |
-
prod *= i
|
38 |
-
return 1.0 / prod
|
39 |
-
|
40 |
-
|
41 |
-
def KVal(M, L):
|
42 |
-
return math.sqrt(((2 * L + 1) / (4 * math.pi)) * (factratio(L - M, L + M)))
|
43 |
-
|
44 |
-
|
45 |
-
def AssociatedLegendre(M, L, x):
|
46 |
-
if M < 0 or M > L or np.max(np.abs(x)) > 1.0:
|
47 |
-
return np.zeros_like(x)
|
48 |
-
|
49 |
-
pmm = np.ones_like(x)
|
50 |
-
if M > 0:
|
51 |
-
somx2 = np.sqrt((1.0 + x) * (1.0 - x))
|
52 |
-
fact = 1.0
|
53 |
-
for i in range(1, M + 1):
|
54 |
-
pmm = -pmm * fact * somx2
|
55 |
-
fact = fact + 2
|
56 |
-
|
57 |
-
if L == M:
|
58 |
-
return pmm
|
59 |
-
else:
|
60 |
-
pmmp1 = x * (2 * M + 1) * pmm
|
61 |
-
if L == M + 1:
|
62 |
-
return pmmp1
|
63 |
-
else:
|
64 |
-
pll = np.zeros_like(x)
|
65 |
-
for i in range(M + 2, L + 1):
|
66 |
-
pll = (x * (2 * i - 1) * pmmp1 - (i + M - 1) * pmm) / (i - M)
|
67 |
-
pmm = pmmp1
|
68 |
-
pmmp1 = pll
|
69 |
-
return pll
|
70 |
-
|
71 |
-
|
72 |
-
def SphericalHarmonic(M, L, theta, phi):
|
73 |
-
if M > 0:
|
74 |
-
return math.sqrt(2.0) * KVal(M, L) * np.cos(
|
75 |
-
M * phi) * AssociatedLegendre(M, L, np.cos(theta))
|
76 |
-
elif M < 0:
|
77 |
-
return math.sqrt(2.0) * KVal(-M, L) * np.sin(
|
78 |
-
-M * phi) * AssociatedLegendre(-M, L, np.cos(theta))
|
79 |
-
else:
|
80 |
-
return KVal(0, L) * AssociatedLegendre(0, L, np.cos(theta))
|
81 |
-
|
82 |
-
|
83 |
-
def save_obj(mesh_path, verts):
|
84 |
-
file = open(mesh_path, 'w')
|
85 |
-
for v in verts:
|
86 |
-
file.write('v %.4f %.4f %.4f\n' % (v[0], v[1], v[2]))
|
87 |
-
file.close()
|
88 |
-
|
89 |
-
|
90 |
-
def sampleSphericalDirections(n):
|
91 |
-
xv = np.random.rand(n, n)
|
92 |
-
yv = np.random.rand(n, n)
|
93 |
-
theta = np.arccos(1 - 2 * xv)
|
94 |
-
phi = 2.0 * math.pi * yv
|
95 |
-
|
96 |
-
phi = phi.reshape(-1)
|
97 |
-
theta = theta.reshape(-1)
|
98 |
-
|
99 |
-
vx = -np.sin(theta) * np.cos(phi)
|
100 |
-
vy = -np.sin(theta) * np.sin(phi)
|
101 |
-
vz = np.cos(theta)
|
102 |
-
return np.stack([vx, vy, vz], 1), phi, theta
|
103 |
-
|
104 |
-
|
105 |
-
def getSHCoeffs(order, phi, theta):
|
106 |
-
shs = []
|
107 |
-
for n in range(0, order + 1):
|
108 |
-
for m in range(-n, n + 1):
|
109 |
-
s = SphericalHarmonic(m, n, theta, phi)
|
110 |
-
shs.append(s)
|
111 |
-
|
112 |
-
return np.stack(shs, 1)
|
113 |
-
|
114 |
-
|
115 |
-
def computePRT(mesh_path, scale, n, order):
|
116 |
-
|
117 |
-
prt_dir = os.path.join(os.path.dirname(mesh_path), "prt")
|
118 |
-
bounce_path = os.path.join(prt_dir, "bounce.npy")
|
119 |
-
face_path = os.path.join(prt_dir, "face.npy")
|
120 |
-
|
121 |
-
os.makedirs(prt_dir, exist_ok=True)
|
122 |
-
|
123 |
-
PRT = None
|
124 |
-
F = None
|
125 |
-
|
126 |
-
if os.path.exists(bounce_path) and os.path.exists(face_path):
|
127 |
-
|
128 |
-
PRT = np.load(bounce_path)
|
129 |
-
F = np.load(face_path)
|
130 |
-
|
131 |
-
else:
|
132 |
-
|
133 |
-
mesh = trimesh.load(mesh_path,
|
134 |
-
skip_materials=True,
|
135 |
-
process=False,
|
136 |
-
maintain_order=True)
|
137 |
-
mesh.vertices *= scale
|
138 |
-
|
139 |
-
vectors_orig, phi, theta = sampleSphericalDirections(n)
|
140 |
-
SH_orig = getSHCoeffs(order, phi, theta)
|
141 |
-
|
142 |
-
w = 4.0 * math.pi / (n * n)
|
143 |
-
|
144 |
-
origins = mesh.vertices
|
145 |
-
normals = mesh.vertex_normals
|
146 |
-
n_v = origins.shape[0]
|
147 |
-
|
148 |
-
origins = np.repeat(origins[:, None], n, axis=1).reshape(-1, 3)
|
149 |
-
normals = np.repeat(normals[:, None], n, axis=1).reshape(-1, 3)
|
150 |
-
PRT_all = None
|
151 |
-
for i in range(n):
|
152 |
-
SH = np.repeat(SH_orig[None, (i * n):((i + 1) * n)], n_v,
|
153 |
-
axis=0).reshape(-1, SH_orig.shape[1])
|
154 |
-
vectors = np.repeat(vectors_orig[None, (i * n):((i + 1) * n)],
|
155 |
-
n_v,
|
156 |
-
axis=0).reshape(-1, 3)
|
157 |
-
|
158 |
-
dots = (vectors * normals).sum(1)
|
159 |
-
front = (dots > 0.0)
|
160 |
-
|
161 |
-
delta = 1e-3 * min(mesh.bounding_box.extents)
|
162 |
-
|
163 |
-
hits = mesh.ray.intersects_any(origins + delta * normals, vectors)
|
164 |
-
nohits = np.logical_and(front, np.logical_not(hits))
|
165 |
-
|
166 |
-
PRT = (nohits.astype(np.float) * dots)[:, None] * SH
|
167 |
-
|
168 |
-
if PRT_all is not None:
|
169 |
-
PRT_all += (PRT.reshape(-1, n, SH.shape[1]).sum(1))
|
170 |
-
else:
|
171 |
-
PRT_all = (PRT.reshape(-1, n, SH.shape[1]).sum(1))
|
172 |
-
|
173 |
-
PRT = w * PRT_all
|
174 |
-
F = mesh.faces
|
175 |
-
|
176 |
-
np.save(bounce_path, PRT)
|
177 |
-
np.save(face_path, F)
|
178 |
-
|
179 |
-
# NOTE: trimesh sometimes break the original vertex order, but topology will not change.
|
180 |
-
# when loading PRT in other program, use the triangle list from trimesh.
|
181 |
-
|
182 |
-
return PRT, F
|
183 |
-
|
184 |
-
|
185 |
-
def testPRT(obj_path, n=40):
|
186 |
-
|
187 |
-
os.makedirs(os.path.join(os.path.dirname(obj_path),
|
188 |
-
f'../bounce/{os.path.basename(obj_path)[:-4]}'),
|
189 |
-
exist_ok=True)
|
190 |
-
|
191 |
-
PRT, F = computePRT(obj_path, n, 2)
|
192 |
-
np.savetxt(
|
193 |
-
os.path.join(os.path.dirname(obj_path),
|
194 |
-
f'../bounce/{os.path.basename(obj_path)[:-4]}',
|
195 |
-
'bounce.npy'), PRT)
|
196 |
-
np.save(
|
197 |
-
os.path.join(os.path.dirname(obj_path),
|
198 |
-
f'../bounce/{os.path.basename(obj_path)[:-4]}',
|
199 |
-
'face.npy'), F)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib /common / __init__.py
DELETED
File without changes
|
lib /common / seg3d_utils.py
DELETED
@@ -1,390 +0,0 @@
|
|
1 |
-
|
2 |
-
# -*- coding: utf-8 -*-
|
3 |
-
|
4 |
-
# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
|
5 |
-
# holder of all proprietary rights on this computer program.
|
6 |
-
# You can only use this computer program if you have closed
|
7 |
-
# a license agreement with MPG or you get the right to use the computer
|
8 |
-
# program from someone who is authorized to grant you that right.
|
9 |
-
# Any use of the computer program without a valid license is prohibited and
|
10 |
-
# liable to prosecution.
|
11 |
-
#
|
12 |
-
# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
|
13 |
-
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
|
14 |
-
# for Intelligent Systems. All rights reserved.
|
15 |
-
#
|
16 |
-
# Contact: [email protected]
|
17 |
-
|
18 |
-
import torch
|
19 |
-
import torch.nn as nn
|
20 |
-
import torch.nn.functional as F
|
21 |
-
import matplotlib.pyplot as plt
|
22 |
-
|
23 |
-
|
24 |
-
def plot_mask2D(mask,
|
25 |
-
title="",
|
26 |
-
point_coords=None,
|
27 |
-
figsize=10,
|
28 |
-
point_marker_size=5):
|
29 |
-
'''
|
30 |
-
Simple plotting tool to show intermediate mask predictions and points
|
31 |
-
where PointRend is applied.
|
32 |
-
Args:
|
33 |
-
mask (Tensor): mask prediction of shape HxW
|
34 |
-
title (str): title for the plot
|
35 |
-
point_coords ((Tensor, Tensor)): x and y point coordinates
|
36 |
-
figsize (int): size of the figure to plot
|
37 |
-
point_marker_size (int): marker size for points
|
38 |
-
'''
|
39 |
-
|
40 |
-
H, W = mask.shape
|
41 |
-
plt.figure(figsize=(figsize, figsize))
|
42 |
-
if title:
|
43 |
-
title += ", "
|
44 |
-
plt.title("{}resolution {}x{}".format(title, H, W), fontsize=30)
|
45 |
-
plt.ylabel(H, fontsize=30)
|
46 |
-
plt.xlabel(W, fontsize=30)
|
47 |
-
plt.xticks([], [])
|
48 |
-
plt.yticks([], [])
|
49 |
-
plt.imshow(mask.detach(),
|
50 |
-
interpolation="nearest",
|
51 |
-
cmap=plt.get_cmap('gray'))
|
52 |
-
if point_coords is not None:
|
53 |
-
plt.scatter(x=point_coords[0],
|
54 |
-
y=point_coords[1],
|
55 |
-
color="red",
|
56 |
-
s=point_marker_size,
|
57 |
-
clip_on=True)
|
58 |
-
plt.xlim(-0.5, W - 0.5)
|
59 |
-
plt.ylim(H - 0.5, -0.5)
|
60 |
-
plt.show()
|
61 |
-
|
62 |
-
|
63 |
-
def plot_mask3D(mask=None,
|
64 |
-
title="",
|
65 |
-
point_coords=None,
|
66 |
-
figsize=1500,
|
67 |
-
point_marker_size=8,
|
68 |
-
interactive=True):
|
69 |
-
'''
|
70 |
-
Simple plotting tool to show intermediate mask predictions and points
|
71 |
-
where PointRend is applied.
|
72 |
-
Args:
|
73 |
-
mask (Tensor): mask prediction of shape DxHxW
|
74 |
-
title (str): title for the plot
|
75 |
-
point_coords ((Tensor, Tensor, Tensor)): x and y and z point coordinates
|
76 |
-
figsize (int): size of the figure to plot
|
77 |
-
point_marker_size (int): marker size for points
|
78 |
-
'''
|
79 |
-
import trimesh
|
80 |
-
import vtkplotter
|
81 |
-
from skimage import measure
|
82 |
-
|
83 |
-
vp = vtkplotter.Plotter(title=title, size=(figsize, figsize))
|
84 |
-
vis_list = []
|
85 |
-
|
86 |
-
if mask is not None:
|
87 |
-
mask = mask.detach().to("cpu").numpy()
|
88 |
-
mask = mask.transpose(2, 1, 0)
|
89 |
-
|
90 |
-
# marching cube to find surface
|
91 |
-
verts, faces, normals, values = measure.marching_cubes_lewiner(
|
92 |
-
mask, 0.5, gradient_direction='ascent')
|
93 |
-
|
94 |
-
# create a mesh
|
95 |
-
mesh = trimesh.Trimesh(verts, faces)
|
96 |
-
mesh.visual.face_colors = [200, 200, 250, 100]
|
97 |
-
vis_list.append(mesh)
|
98 |
-
|
99 |
-
if point_coords is not None:
|
100 |
-
point_coords = torch.stack(point_coords, 1).to("cpu").numpy()
|
101 |
-
|
102 |
-
# import numpy as np
|
103 |
-
# select_x = np.logical_and(point_coords[:, 0] >= 16, point_coords[:, 0] <= 112)
|
104 |
-
# select_y = np.logical_and(point_coords[:, 1] >= 48, point_coords[:, 1] <= 272)
|
105 |
-
# select_z = np.logical_and(point_coords[:, 2] >= 16, point_coords[:, 2] <= 112)
|
106 |
-
# select = np.logical_and(np.logical_and(select_x, select_y), select_z)
|
107 |
-
# point_coords = point_coords[select, :]
|
108 |
-
|
109 |
-
pc = vtkplotter.Points(point_coords, r=point_marker_size, c='red')
|
110 |
-
vis_list.append(pc)
|
111 |
-
|
112 |
-
vp.show(*vis_list,
|
113 |
-
bg="white",
|
114 |
-
axes=1,
|
115 |
-
interactive=interactive,
|
116 |
-
azimuth=30,
|
117 |
-
elevation=30)
|
118 |
-
|
119 |
-
|
120 |
-
def create_grid3D(min, max, steps):
|
121 |
-
if type(min) is int:
|
122 |
-
min = (min, min, min) # (x, y, z)
|
123 |
-
if type(max) is int:
|
124 |
-
max = (max, max, max) # (x, y)
|
125 |
-
if type(steps) is int:
|
126 |
-
steps = (steps, steps, steps) # (x, y, z)
|
127 |
-
arrangeX = torch.linspace(min[0], max[0], steps[0]).long()
|
128 |
-
arrangeY = torch.linspace(min[1], max[1], steps[1]).long()
|
129 |
-
arrangeZ = torch.linspace(min[2], max[2], steps[2]).long()
|
130 |
-
gridD, girdH, gridW = torch.meshgrid([arrangeZ, arrangeY, arrangeX])
|
131 |
-
coords = torch.stack([gridW, girdH,
|
132 |
-
gridD]) # [2, steps[0], steps[1], steps[2]]
|
133 |
-
coords = coords.view(3, -1).t() # [N, 3]
|
134 |
-
return coords
|
135 |
-
|
136 |
-
|
137 |
-
def create_grid2D(min, max, steps):
|
138 |
-
if type(min) is int:
|
139 |
-
min = (min, min) # (x, y)
|
140 |
-
if type(max) is int:
|
141 |
-
max = (max, max) # (x, y)
|
142 |
-
if type(steps) is int:
|
143 |
-
steps = (steps, steps) # (x, y)
|
144 |
-
arrangeX = torch.linspace(min[0], max[0], steps[0]).long()
|
145 |
-
arrangeY = torch.linspace(min[1], max[1], steps[1]).long()
|
146 |
-
girdH, gridW = torch.meshgrid([arrangeY, arrangeX])
|
147 |
-
coords = torch.stack([gridW, girdH]) # [2, steps[0], steps[1]]
|
148 |
-
coords = coords.view(2, -1).t() # [N, 2]
|
149 |
-
return coords
|
150 |
-
|
151 |
-
|
152 |
-
class SmoothConv2D(nn.Module):
|
153 |
-
def __init__(self, in_channels, out_channels, kernel_size=3):
|
154 |
-
super().__init__()
|
155 |
-
assert kernel_size % 2 == 1, "kernel_size for smooth_conv must be odd: {3, 5, ...}"
|
156 |
-
self.padding = (kernel_size - 1) // 2
|
157 |
-
|
158 |
-
weight = torch.ones(
|
159 |
-
(in_channels, out_channels, kernel_size, kernel_size),
|
160 |
-
dtype=torch.float32) / (kernel_size**2)
|
161 |
-
self.register_buffer('weight', weight)
|
162 |
-
|
163 |
-
def forward(self, input):
|
164 |
-
return F.conv2d(input, self.weight, padding=self.padding)
|
165 |
-
|
166 |
-
|
167 |
-
class SmoothConv3D(nn.Module):
|
168 |
-
def __init__(self, in_channels, out_channels, kernel_size=3):
|
169 |
-
super().__init__()
|
170 |
-
assert kernel_size % 2 == 1, "kernel_size for smooth_conv must be odd: {3, 5, ...}"
|
171 |
-
self.padding = (kernel_size - 1) // 2
|
172 |
-
|
173 |
-
weight = torch.ones(
|
174 |
-
(in_channels, out_channels, kernel_size, kernel_size, kernel_size),
|
175 |
-
dtype=torch.float32) / (kernel_size**3)
|
176 |
-
self.register_buffer('weight', weight)
|
177 |
-
|
178 |
-
def forward(self, input):
|
179 |
-
return F.conv3d(input, self.weight, padding=self.padding)
|
180 |
-
|
181 |
-
|
182 |
-
def build_smooth_conv3D(in_channels=1,
|
183 |
-
out_channels=1,
|
184 |
-
kernel_size=3,
|
185 |
-
padding=1):
|
186 |
-
smooth_conv = torch.nn.Conv3d(in_channels=in_channels,
|
187 |
-
out_channels=out_channels,
|
188 |
-
kernel_size=kernel_size,
|
189 |
-
padding=padding)
|
190 |
-
smooth_conv.weight.data = torch.ones(
|
191 |
-
(in_channels, out_channels, kernel_size, kernel_size, kernel_size),
|
192 |
-
dtype=torch.float32) / (kernel_size**3)
|
193 |
-
smooth_conv.bias.data = torch.zeros(out_channels)
|
194 |
-
return smooth_conv
|
195 |
-
|
196 |
-
|
197 |
-
def build_smooth_conv2D(in_channels=1,
|
198 |
-
out_channels=1,
|
199 |
-
kernel_size=3,
|
200 |
-
padding=1):
|
201 |
-
smooth_conv = torch.nn.Conv2d(in_channels=in_channels,
|
202 |
-
out_channels=out_channels,
|
203 |
-
kernel_size=kernel_size,
|
204 |
-
padding=padding)
|
205 |
-
smooth_conv.weight.data = torch.ones(
|
206 |
-
(in_channels, out_channels, kernel_size, kernel_size),
|
207 |
-
dtype=torch.float32) / (kernel_size**2)
|
208 |
-
smooth_conv.bias.data = torch.zeros(out_channels)
|
209 |
-
return smooth_conv
|
210 |
-
|
211 |
-
|
212 |
-
def get_uncertain_point_coords_on_grid3D(uncertainty_map, num_points,
|
213 |
-
**kwargs):
|
214 |
-
"""
|
215 |
-
Find `num_points` most uncertain points from `uncertainty_map` grid.
|
216 |
-
Args:
|
217 |
-
uncertainty_map (Tensor): A tensor of shape (N, 1, H, W, D) that contains uncertainty
|
218 |
-
values for a set of points on a regular H x W x D grid.
|
219 |
-
num_points (int): The number of points P to select.
|
220 |
-
Returns:
|
221 |
-
point_indices (Tensor): A tensor of shape (N, P) that contains indices from
|
222 |
-
[0, H x W x D) of the most uncertain points.
|
223 |
-
point_coords (Tensor): A tensor of shape (N, P, 3) that contains [0, 1] x [0, 1] normalized
|
224 |
-
coordinates of the most uncertain points from the H x W x D grid.
|
225 |
-
"""
|
226 |
-
R, _, D, H, W = uncertainty_map.shape
|
227 |
-
# h_step = 1.0 / float(H)
|
228 |
-
# w_step = 1.0 / float(W)
|
229 |
-
# d_step = 1.0 / float(D)
|
230 |
-
|
231 |
-
num_points = min(D * H * W, num_points)
|
232 |
-
point_scores, point_indices = torch.topk(uncertainty_map.view(
|
233 |
-
R, D * H * W),
|
234 |
-
k=num_points,
|
235 |
-
dim=1)
|
236 |
-
point_coords = torch.zeros(R,
|
237 |
-
num_points,
|
238 |
-
3,
|
239 |
-
dtype=torch.float,
|
240 |
-
device=uncertainty_map.device)
|
241 |
-
# point_coords[:, :, 0] = h_step / 2.0 + (point_indices // (W * D)).to(torch.float) * h_step
|
242 |
-
# point_coords[:, :, 1] = w_step / 2.0 + (point_indices % (W * D) // D).to(torch.float) * w_step
|
243 |
-
# point_coords[:, :, 2] = d_step / 2.0 + (point_indices % D).to(torch.float) * d_step
|
244 |
-
point_coords[:, :, 0] = (point_indices % W).to(torch.float) # x
|
245 |
-
point_coords[:, :, 1] = (point_indices % (H * W) // W).to(torch.float) # y
|
246 |
-
point_coords[:, :, 2] = (point_indices // (H * W)).to(torch.float) # z
|
247 |
-
print(f"resolution {D} x {H} x {W}", point_scores.min(),
|
248 |
-
point_scores.max())
|
249 |
-
return point_indices, point_coords
|
250 |
-
|
251 |
-
|
252 |
-
def get_uncertain_point_coords_on_grid3D_faster(uncertainty_map, num_points,
|
253 |
-
clip_min):
|
254 |
-
"""
|
255 |
-
Find `num_points` most uncertain points from `uncertainty_map` grid.
|
256 |
-
Args:
|
257 |
-
uncertainty_map (Tensor): A tensor of shape (N, 1, H, W, D) that contains uncertainty
|
258 |
-
values for a set of points on a regular H x W x D grid.
|
259 |
-
num_points (int): The number of points P to select.
|
260 |
-
Returns:
|
261 |
-
point_indices (Tensor): A tensor of shape (N, P) that contains indices from
|
262 |
-
[0, H x W x D) of the most uncertain points.
|
263 |
-
point_coords (Tensor): A tensor of shape (N, P, 3) that contains [0, 1] x [0, 1] normalized
|
264 |
-
coordinates of the most uncertain points from the H x W x D grid.
|
265 |
-
"""
|
266 |
-
R, _, D, H, W = uncertainty_map.shape
|
267 |
-
# h_step = 1.0 / float(H)
|
268 |
-
# w_step = 1.0 / float(W)
|
269 |
-
# d_step = 1.0 / float(D)
|
270 |
-
|
271 |
-
assert R == 1, "batchsize > 1 is not implemented!"
|
272 |
-
uncertainty_map = uncertainty_map.view(D * H * W)
|
273 |
-
indices = (uncertainty_map >= clip_min).nonzero().squeeze(1)
|
274 |
-
num_points = min(num_points, indices.size(0))
|
275 |
-
point_scores, point_indices = torch.topk(uncertainty_map[indices],
|
276 |
-
k=num_points,
|
277 |
-
dim=0)
|
278 |
-
point_indices = indices[point_indices].unsqueeze(0)
|
279 |
-
|
280 |
-
point_coords = torch.zeros(R,
|
281 |
-
num_points,
|
282 |
-
3,
|
283 |
-
dtype=torch.float,
|
284 |
-
device=uncertainty_map.device)
|
285 |
-
# point_coords[:, :, 0] = h_step / 2.0 + (point_indices // (W * D)).to(torch.float) * h_step
|
286 |
-
# point_coords[:, :, 1] = w_step / 2.0 + (point_indices % (W * D) // D).to(torch.float) * w_step
|
287 |
-
# point_coords[:, :, 2] = d_step / 2.0 + (point_indices % D).to(torch.float) * d_step
|
288 |
-
point_coords[:, :, 0] = (point_indices % W).to(torch.float) # x
|
289 |
-
point_coords[:, :, 1] = (point_indices % (H * W) // W).to(torch.float) # y
|
290 |
-
point_coords[:, :, 2] = (point_indices // (H * W)).to(torch.float) # z
|
291 |
-
# print (f"resolution {D} x {H} x {W}", point_scores.min(), point_scores.max())
|
292 |
-
return point_indices, point_coords
|
293 |
-
|
294 |
-
|
295 |
-
def get_uncertain_point_coords_on_grid2D(uncertainty_map, num_points,
|
296 |
-
**kwargs):
|
297 |
-
"""
|
298 |
-
Find `num_points` most uncertain points from `uncertainty_map` grid.
|
299 |
-
Args:
|
300 |
-
uncertainty_map (Tensor): A tensor of shape (N, 1, H, W) that contains uncertainty
|
301 |
-
values for a set of points on a regular H x W grid.
|
302 |
-
num_points (int): The number of points P to select.
|
303 |
-
Returns:
|
304 |
-
point_indices (Tensor): A tensor of shape (N, P) that contains indices from
|
305 |
-
[0, H x W) of the most uncertain points.
|
306 |
-
point_coords (Tensor): A tensor of shape (N, P, 2) that contains [0, 1] x [0, 1] normalized
|
307 |
-
coordinates of the most uncertain points from the H x W grid.
|
308 |
-
"""
|
309 |
-
R, _, H, W = uncertainty_map.shape
|
310 |
-
# h_step = 1.0 / float(H)
|
311 |
-
# w_step = 1.0 / float(W)
|
312 |
-
|
313 |
-
num_points = min(H * W, num_points)
|
314 |
-
point_scores, point_indices = torch.topk(uncertainty_map.view(R, H * W),
|
315 |
-
k=num_points,
|
316 |
-
dim=1)
|
317 |
-
point_coords = torch.zeros(R,
|
318 |
-
num_points,
|
319 |
-
2,
|
320 |
-
dtype=torch.long,
|
321 |
-
device=uncertainty_map.device)
|
322 |
-
# point_coords[:, :, 0] = w_step / 2.0 + (point_indices % W).to(torch.float) * w_step
|
323 |
-
# point_coords[:, :, 1] = h_step / 2.0 + (point_indices // W).to(torch.float) * h_step
|
324 |
-
point_coords[:, :, 0] = (point_indices % W).to(torch.long)
|
325 |
-
point_coords[:, :, 1] = (point_indices // W).to(torch.long)
|
326 |
-
# print (point_scores.min(), point_scores.max())
|
327 |
-
return point_indices, point_coords
|
328 |
-
|
329 |
-
|
330 |
-
def get_uncertain_point_coords_on_grid2D_faster(uncertainty_map, num_points,
|
331 |
-
clip_min):
|
332 |
-
"""
|
333 |
-
Find `num_points` most uncertain points from `uncertainty_map` grid.
|
334 |
-
Args:
|
335 |
-
uncertainty_map (Tensor): A tensor of shape (N, 1, H, W) that contains uncertainty
|
336 |
-
values for a set of points on a regular H x W grid.
|
337 |
-
num_points (int): The number of points P to select.
|
338 |
-
Returns:
|
339 |
-
point_indices (Tensor): A tensor of shape (N, P) that contains indices from
|
340 |
-
[0, H x W) of the most uncertain points.
|
341 |
-
point_coords (Tensor): A tensor of shape (N, P, 2) that contains [0, 1] x [0, 1] normalized
|
342 |
-
coordinates of the most uncertain points from the H x W grid.
|
343 |
-
"""
|
344 |
-
R, _, H, W = uncertainty_map.shape
|
345 |
-
# h_step = 1.0 / float(H)
|
346 |
-
# w_step = 1.0 / float(W)
|
347 |
-
|
348 |
-
assert R == 1, "batchsize > 1 is not implemented!"
|
349 |
-
uncertainty_map = uncertainty_map.view(H * W)
|
350 |
-
indices = (uncertainty_map >= clip_min).nonzero().squeeze(1)
|
351 |
-
num_points = min(num_points, indices.size(0))
|
352 |
-
point_scores, point_indices = torch.topk(uncertainty_map[indices],
|
353 |
-
k=num_points,
|
354 |
-
dim=0)
|
355 |
-
point_indices = indices[point_indices].unsqueeze(0)
|
356 |
-
|
357 |
-
point_coords = torch.zeros(R,
|
358 |
-
num_points,
|
359 |
-
2,
|
360 |
-
dtype=torch.long,
|
361 |
-
device=uncertainty_map.device)
|
362 |
-
# point_coords[:, :, 0] = w_step / 2.0 + (point_indices % W).to(torch.float) * w_step
|
363 |
-
# point_coords[:, :, 1] = h_step / 2.0 + (point_indices // W).to(torch.float) * h_step
|
364 |
-
point_coords[:, :, 0] = (point_indices % W).to(torch.long)
|
365 |
-
point_coords[:, :, 1] = (point_indices // W).to(torch.long)
|
366 |
-
# print (point_scores.min(), point_scores.max())
|
367 |
-
return point_indices, point_coords
|
368 |
-
|
369 |
-
|
370 |
-
def calculate_uncertainty(logits, classes=None, balance_value=0.5):
|
371 |
-
"""
|
372 |
-
We estimate uncerainty as L1 distance between 0.0 and the logit prediction in 'logits' for the
|
373 |
-
foreground class in `classes`.
|
374 |
-
Args:
|
375 |
-
logits (Tensor): A tensor of shape (R, C, ...) or (R, 1, ...) for class-specific or
|
376 |
-
class-agnostic, where R is the total number of predicted masks in all images and C is
|
377 |
-
the number of foreground classes. The values are logits.
|
378 |
-
classes (list): A list of length R that contains either predicted of ground truth class
|
379 |
-
for eash predicted mask.
|
380 |
-
Returns:
|
381 |
-
scores (Tensor): A tensor of shape (R, 1, ...) that contains uncertainty scores with
|
382 |
-
the most uncertain locations having the highest uncertainty score.
|
383 |
-
"""
|
384 |
-
if logits.shape[1] == 1:
|
385 |
-
gt_class_logits = logits
|
386 |
-
else:
|
387 |
-
gt_class_logits = logits[
|
388 |
-
torch.arange(logits.shape[0], device=logits.device),
|
389 |
-
classes].unsqueeze(1)
|
390 |
-
return -torch.abs(gt_class_logits - balance_value)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib /common /cloth_extraction.py
DELETED
@@ -1,170 +0,0 @@
|
|
1 |
-
import numpy as np
|
2 |
-
import json
|
3 |
-
import os
|
4 |
-
import itertools
|
5 |
-
import trimesh
|
6 |
-
from matplotlib.path import Path
|
7 |
-
from collections import Counter
|
8 |
-
from sklearn.neighbors import KNeighborsClassifier
|
9 |
-
|
10 |
-
|
11 |
-
def load_segmentation(path, shape):
|
12 |
-
"""
|
13 |
-
Get a segmentation mask for a given image
|
14 |
-
Arguments:
|
15 |
-
path: path to the segmentation json file
|
16 |
-
shape: shape of the output mask
|
17 |
-
Returns:
|
18 |
-
Returns a segmentation mask
|
19 |
-
"""
|
20 |
-
with open(path) as json_file:
|
21 |
-
dict = json.load(json_file)
|
22 |
-
segmentations = []
|
23 |
-
for key, val in dict.items():
|
24 |
-
if not key.startswith('item'):
|
25 |
-
continue
|
26 |
-
|
27 |
-
# Each item can have multiple polygons. Combine them to one
|
28 |
-
# segmentation_coord = list(itertools.chain.from_iterable(val['segmentation']))
|
29 |
-
# segmentation_coord = np.round(np.array(segmentation_coord)).astype(int)
|
30 |
-
|
31 |
-
coordinates = []
|
32 |
-
for segmentation_coord in val['segmentation']:
|
33 |
-
# The format before is [x1,y1, x2, y2, ....]
|
34 |
-
x = segmentation_coord[::2]
|
35 |
-
y = segmentation_coord[1::2]
|
36 |
-
xy = np.vstack((x, y)).T
|
37 |
-
coordinates.append(xy)
|
38 |
-
|
39 |
-
segmentations.append(
|
40 |
-
{'type': val['category_name'], 'type_id': val['category_id'], 'coordinates': coordinates})
|
41 |
-
|
42 |
-
return segmentations
|
43 |
-
|
44 |
-
|
45 |
-
def smpl_to_recon_labels(recon, smpl, k=1):
|
46 |
-
"""
|
47 |
-
Get the bodypart labels for the recon object by using the labels from the corresponding smpl object
|
48 |
-
Arguments:
|
49 |
-
recon: trimesh object (fully clothed model)
|
50 |
-
shape: trimesh object (smpl model)
|
51 |
-
k: number of nearest neighbours to use
|
52 |
-
Returns:
|
53 |
-
Returns a dictionary containing the bodypart and the corresponding indices
|
54 |
-
"""
|
55 |
-
smpl_vert_segmentation = json.load(
|
56 |
-
open(os.path.join(os.path.dirname(__file__), 'smpl_vert_segmentation.json')))
|
57 |
-
n = smpl.vertices.shape[0]
|
58 |
-
y = np.array([None] * n)
|
59 |
-
for key, val in smpl_vert_segmentation.items():
|
60 |
-
y[val] = key
|
61 |
-
|
62 |
-
classifier = KNeighborsClassifier(n_neighbors=1)
|
63 |
-
classifier.fit(smpl.vertices, y)
|
64 |
-
|
65 |
-
y_pred = classifier.predict(recon.vertices)
|
66 |
-
|
67 |
-
recon_labels = {}
|
68 |
-
for key in smpl_vert_segmentation.keys():
|
69 |
-
recon_labels[key] = list(np.argwhere(
|
70 |
-
y_pred == key).flatten().astype(int))
|
71 |
-
|
72 |
-
return recon_labels
|
73 |
-
|
74 |
-
|
75 |
-
def extract_cloth(recon, segmentation, K, R, t, smpl=None):
|
76 |
-
"""
|
77 |
-
Extract a portion of a mesh using 2d segmentation coordinates
|
78 |
-
Arguments:
|
79 |
-
recon: fully clothed mesh
|
80 |
-
seg_coord: segmentation coordinates in 2D (NDC)
|
81 |
-
K: intrinsic matrix of the projection
|
82 |
-
R: rotation matrix of the projection
|
83 |
-
t: translation vector of the projection
|
84 |
-
Returns:
|
85 |
-
Returns a submesh using the segmentation coordinates
|
86 |
-
"""
|
87 |
-
seg_coord = segmentation['coord_normalized']
|
88 |
-
mesh = trimesh.Trimesh(recon.vertices, recon.faces)
|
89 |
-
extrinsic = np.zeros((3, 4))
|
90 |
-
extrinsic[:3, :3] = R
|
91 |
-
extrinsic[:, 3] = t
|
92 |
-
P = K[:3, :3] @ extrinsic
|
93 |
-
|
94 |
-
P_inv = np.linalg.pinv(P)
|
95 |
-
|
96 |
-
# Each segmentation can contain multiple polygons
|
97 |
-
# We need to check them separately
|
98 |
-
points_so_far = []
|
99 |
-
faces = recon.faces
|
100 |
-
for polygon in seg_coord:
|
101 |
-
n = len(polygon)
|
102 |
-
coords_h = np.hstack((polygon, np.ones((n, 1))))
|
103 |
-
# Apply the inverse projection on homogeneus 2D coordinates to get the corresponding 3d Coordinates
|
104 |
-
XYZ = P_inv @ coords_h[:, :, None]
|
105 |
-
XYZ = XYZ.reshape((XYZ.shape[0], XYZ.shape[1]))
|
106 |
-
XYZ = XYZ[:, :3] / XYZ[:, 3, None]
|
107 |
-
|
108 |
-
p = Path(XYZ[:, :2])
|
109 |
-
|
110 |
-
grid = p.contains_points(recon.vertices[:, :2])
|
111 |
-
indeces = np.argwhere(grid == True)
|
112 |
-
points_so_far += list(indeces.flatten())
|
113 |
-
|
114 |
-
if smpl is not None:
|
115 |
-
num_verts = recon.vertices.shape[0]
|
116 |
-
recon_labels = smpl_to_recon_labels(recon, smpl)
|
117 |
-
body_parts_to_remove = ['rightHand', 'leftToeBase', 'leftFoot', 'rightFoot', 'head',
|
118 |
-
'leftHandIndex1', 'rightHandIndex1', 'rightToeBase', 'leftHand', 'rightHand']
|
119 |
-
type = segmentation['type_id']
|
120 |
-
|
121 |
-
# Remove additional bodyparts that are most likely not part of the segmentation but might intersect (e.g. hand in front of torso)
|
122 |
-
# https://github.com/switchablenorms/DeepFashion2
|
123 |
-
# Short sleeve clothes
|
124 |
-
if type == 1 or type == 3 or type == 10:
|
125 |
-
body_parts_to_remove += ['leftForeArm', 'rightForeArm']
|
126 |
-
# No sleeves at all or lower body clothes
|
127 |
-
elif type == 5 or type == 6 or type == 12 or type == 13 or type == 8 or type == 9:
|
128 |
-
body_parts_to_remove += ['leftForeArm',
|
129 |
-
'rightForeArm', 'leftArm', 'rightArm']
|
130 |
-
# Shorts
|
131 |
-
elif type == 7:
|
132 |
-
body_parts_to_remove += ['leftLeg', 'rightLeg',
|
133 |
-
'leftForeArm', 'rightForeArm', 'leftArm', 'rightArm']
|
134 |
-
|
135 |
-
verts_to_remove = list(itertools.chain.from_iterable(
|
136 |
-
[recon_labels[part] for part in body_parts_to_remove]))
|
137 |
-
|
138 |
-
label_mask = np.zeros(num_verts, dtype=bool)
|
139 |
-
label_mask[verts_to_remove] = True
|
140 |
-
|
141 |
-
seg_mask = np.zeros(num_verts, dtype=bool)
|
142 |
-
seg_mask[points_so_far] = True
|
143 |
-
|
144 |
-
# Remove points that belong to other bodyparts
|
145 |
-
# If a vertice in pointsSoFar is included in the bodyparts to remove, then these points should be removed
|
146 |
-
extra_verts_to_remove = np.array(list(seg_mask) and list(label_mask))
|
147 |
-
|
148 |
-
combine_mask = np.zeros(num_verts, dtype=bool)
|
149 |
-
combine_mask[points_so_far] = True
|
150 |
-
combine_mask[extra_verts_to_remove] = False
|
151 |
-
|
152 |
-
all_indices = np.argwhere(combine_mask == True).flatten()
|
153 |
-
|
154 |
-
i_x = np.where(np.in1d(faces[:, 0], all_indices))[0]
|
155 |
-
i_y = np.where(np.in1d(faces[:, 1], all_indices))[0]
|
156 |
-
i_z = np.where(np.in1d(faces[:, 2], all_indices))[0]
|
157 |
-
|
158 |
-
faces_to_keep = np.array(list(set(i_x).union(i_y).union(i_z)))
|
159 |
-
mask = np.zeros(len(recon.faces), dtype=bool)
|
160 |
-
if len(faces_to_keep) > 0:
|
161 |
-
mask[faces_to_keep] = True
|
162 |
-
|
163 |
-
mesh.update_faces(mask)
|
164 |
-
mesh.remove_unreferenced_vertices()
|
165 |
-
|
166 |
-
# mesh.rezero()
|
167 |
-
|
168 |
-
return mesh
|
169 |
-
|
170 |
-
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
lib /common /config.py
DELETED
@@ -1,218 +0,0 @@
|
|
1 |
-
|
2 |
-
# -*- coding: utf-8 -*-
|
3 |
-
|
4 |
-
# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
|
5 |
-
# holder of all proprietary rights on this computer program.
|
6 |
-
# You can only use this computer program if you have closed
|
7 |
-
# a license agreement with MPG or you get the right to use the computer
|
8 |
-
# program from someone who is authorized to grant you that right.
|
9 |
-
# Any use of the computer program without a valid license is prohibited and
|
10 |
-
# liable to prosecution.
|
11 |
-
#
|
12 |
-
# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
|
13 |
-
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
|
14 |
-
# for Intelligent Systems. All rights reserved.
|
15 |
-
#
|
16 |
-
# Contact: [email protected]
|
17 |
-
|
18 |
-
from yacs.config import CfgNode as CN
|
19 |
-
import os
|
20 |
-
|
21 |
-
_C = CN(new_allowed=True)
|
22 |
-
|
23 |
-
# needed by trainer
|
24 |
-
_C.name = 'default'
|
25 |
-
_C.gpus = [0]
|
26 |
-
_C.test_gpus = [1]
|
27 |
-
_C.root = "./data/"
|
28 |
-
_C.ckpt_dir = './data/ckpt/'
|
29 |
-
_C.resume_path = ''
|
30 |
-
_C.normal_path = ''
|
31 |
-
_C.corr_path = ''
|
32 |
-
_C.results_path = './data/results/'
|
33 |
-
_C.projection_mode = 'orthogonal'
|
34 |
-
_C.num_views = 1
|
35 |
-
_C.sdf = False
|
36 |
-
_C.sdf_clip = 5.0
|
37 |
-
|
38 |
-
_C.lr_G = 1e-3
|
39 |
-
_C.lr_C = 1e-3
|
40 |
-
_C.lr_N = 2e-4
|
41 |
-
_C.weight_decay = 0.0
|
42 |
-
_C.momentum = 0.0
|
43 |
-
_C.optim = 'RMSprop'
|
44 |
-
_C.schedule = [5, 10, 15]
|
45 |
-
_C.gamma = 0.1
|
46 |
-
|
47 |
-
_C.overfit = False
|
48 |
-
_C.resume = False
|
49 |
-
_C.test_mode = False
|
50 |
-
_C.test_uv = False
|
51 |
-
_C.draw_geo_thres = 0.60
|
52 |
-
_C.num_sanity_val_steps = 2
|
53 |
-
_C.fast_dev = 0
|
54 |
-
_C.get_fit = False
|
55 |
-
_C.agora = False
|
56 |
-
_C.optim_cloth = False
|
57 |
-
_C.optim_body = False
|
58 |
-
_C.mcube_res = 256
|
59 |
-
_C.clean_mesh = True
|
60 |
-
_C.remesh = False
|
61 |
-
|
62 |
-
_C.batch_size = 4
|
63 |
-
_C.num_threads = 8
|
64 |
-
|
65 |
-
_C.num_epoch = 10
|
66 |
-
_C.freq_plot = 0.01
|
67 |
-
_C.freq_show_train = 0.1
|
68 |
-
_C.freq_show_val = 0.2
|
69 |
-
_C.freq_eval = 0.5
|
70 |
-
_C.accu_grad_batch = 4
|
71 |
-
|
72 |
-
_C.test_items = ['sv', 'mv', 'mv-fusion', 'hybrid', 'dc-pred', 'gt']
|
73 |
-
|
74 |
-
_C.net = CN()
|
75 |
-
_C.net.gtype = 'HGPIFuNet'
|
76 |
-
_C.net.ctype = 'resnet18'
|
77 |
-
_C.net.classifierIMF = 'MultiSegClassifier'
|
78 |
-
_C.net.netIMF = 'resnet18'
|
79 |
-
_C.net.norm = 'group'
|
80 |
-
_C.net.norm_mlp = 'group'
|
81 |
-
_C.net.norm_color = 'group'
|
82 |
-
_C.net.hg_down = 'ave_pool'
|
83 |
-
_C.net.num_views = 1
|
84 |
-
|
85 |
-
# kernel_size, stride, dilation, padding
|
86 |
-
|
87 |
-
_C.net.conv1 = [7, 2, 1, 3]
|
88 |
-
_C.net.conv3x3 = [3, 1, 1, 1]
|
89 |
-
|
90 |
-
_C.net.num_stack = 4
|
91 |
-
_C.net.num_hourglass = 2
|
92 |
-
_C.net.hourglass_dim = 256
|
93 |
-
_C.net.voxel_dim = 32
|
94 |
-
_C.net.resnet_dim = 120
|
95 |
-
_C.net.mlp_dim = [320, 1024, 512, 256, 128, 1]
|
96 |
-
_C.net.mlp_dim_knn = [320, 1024, 512, 256, 128, 3]
|
97 |
-
_C.net.mlp_dim_color = [513, 1024, 512, 256, 128, 3]
|
98 |
-
_C.net.mlp_dim_multiseg = [1088, 2048, 1024, 500]
|
99 |
-
_C.net.res_layers = [2, 3, 4]
|
100 |
-
_C.net.filter_dim = 256
|
101 |
-
_C.net.smpl_dim = 3
|
102 |
-
|
103 |
-
_C.net.cly_dim = 3
|
104 |
-
_C.net.soft_dim = 64
|
105 |
-
_C.net.z_size = 200.0
|
106 |
-
_C.net.N_freqs = 10
|
107 |
-
_C.net.geo_w = 0.1
|
108 |
-
_C.net.norm_w = 0.1
|
109 |
-
_C.net.dc_w = 0.1
|
110 |
-
_C.net.C_cat_to_G = False
|
111 |
-
|
112 |
-
_C.net.skip_hourglass = True
|
113 |
-
_C.net.use_tanh = True
|
114 |
-
_C.net.soft_onehot = True
|
115 |
-
_C.net.no_residual = True
|
116 |
-
_C.net.use_attention = False
|
117 |
-
|
118 |
-
_C.net.prior_type = "sdf"
|
119 |
-
_C.net.smpl_feats = ['sdf', 'cmap', 'norm', 'vis']
|
120 |
-
_C.net.use_filter = True
|
121 |
-
_C.net.use_cc = False
|
122 |
-
_C.net.use_PE = False
|
123 |
-
_C.net.use_IGR = False
|
124 |
-
_C.net.in_geo = ()
|
125 |
-
_C.net.in_nml = ()
|
126 |
-
|
127 |
-
_C.dataset = CN()
|
128 |
-
_C.dataset.root = ''
|
129 |
-
_C.dataset.set_splits = [0.95, 0.04]
|
130 |
-
_C.dataset.types = [
|
131 |
-
"3dpeople", "axyz", "renderpeople", "renderpeople_p27", "humanalloy"
|
132 |
-
]
|
133 |
-
_C.dataset.scales = [1.0, 100.0, 1.0, 1.0, 100.0 / 39.37]
|
134 |
-
_C.dataset.rp_type = "pifu900"
|
135 |
-
_C.dataset.th_type = 'train'
|
136 |
-
_C.dataset.input_size = 512
|
137 |
-
_C.dataset.rotation_num = 3
|
138 |
-
_C.dataset.num_precomp = 10 # Number of segmentation classifiers
|
139 |
-
_C.dataset.num_multiseg = 500 # Number of categories per classifier
|
140 |
-
_C.dataset.num_knn = 10 # for loss/error
|
141 |
-
_C.dataset.num_knn_dis = 20 # for accuracy
|
142 |
-
_C.dataset.num_verts_max = 20000
|
143 |
-
_C.dataset.zray_type = False
|
144 |
-
_C.dataset.online_smpl = False
|
145 |
-
_C.dataset.noise_type = ['z-trans', 'pose', 'beta']
|
146 |
-
_C.dataset.noise_scale = [0.0, 0.0, 0.0]
|
147 |
-
_C.dataset.num_sample_geo = 10000
|
148 |
-
_C.dataset.num_sample_color = 0
|
149 |
-
_C.dataset.num_sample_seg = 0
|
150 |
-
_C.dataset.num_sample_knn = 10000
|
151 |
-
|
152 |
-
_C.dataset.sigma_geo = 5.0
|
153 |
-
_C.dataset.sigma_color = 0.10
|
154 |
-
_C.dataset.sigma_seg = 0.10
|
155 |
-
_C.dataset.thickness_threshold = 20.0
|
156 |
-
_C.dataset.ray_sample_num = 2
|
157 |
-
_C.dataset.semantic_p = False
|
158 |
-
_C.dataset.remove_outlier = False
|
159 |
-
|
160 |
-
_C.dataset.train_bsize = 1.0
|
161 |
-
_C.dataset.val_bsize = 1.0
|
162 |
-
_C.dataset.test_bsize = 1.0
|
163 |
-
|
164 |
-
|
165 |
-
def get_cfg_defaults():
|
166 |
-
"""Get a yacs CfgNode object with default values for my_project."""
|
167 |
-
# Return a clone so that the defaults will not be altered
|
168 |
-
# This is for the "local variable" use pattern
|
169 |
-
return _C.clone()
|
170 |
-
|
171 |
-
|
172 |
-
# Alternatively, provide a way to import the defaults as
|
173 |
-
# a global singleton:
|
174 |
-
cfg = _C # users can `from config import cfg`
|
175 |
-
|
176 |
-
# cfg = get_cfg_defaults()
|
177 |
-
# cfg.merge_from_file('./configs/example.yaml')
|
178 |
-
|
179 |
-
# # Now override from a list (opts could come from the command line)
|
180 |
-
# opts = ['dataset.root', './data/XXXX', 'learning_rate', '1e-2']
|
181 |
-
# cfg.merge_from_list(opts)
|
182 |
-
|
183 |
-
|
184 |
-
def update_cfg(cfg_file):
|
185 |
-
# cfg = get_cfg_defaults()
|
186 |
-
_C.merge_from_file(cfg_file)
|
187 |
-
# return cfg.clone()
|
188 |
-
return _C
|
189 |
-
|
190 |
-
|
191 |
-
def parse_args(args):
|
192 |
-
cfg_file = args.cfg_file
|
193 |
-
if args.cfg_file is not None:
|
194 |
-
cfg = update_cfg(args.cfg_file)
|
195 |
-
else:
|
196 |
-
cfg = get_cfg_defaults()
|
197 |
-
|
198 |
-
# if args.misc is not None:
|
199 |
-
# cfg.merge_from_list(args.misc)
|
200 |
-
|
201 |
-
return cfg
|
202 |
-
|
203 |
-
|
204 |
-
def parse_args_extend(args):
|
205 |
-
if args.resume:
|
206 |
-
if not os.path.exists(args.log_dir):
|
207 |
-
raise ValueError(
|
208 |
-
'Experiment are set to resume mode, but log directory does not exist.'
|
209 |
-
)
|
210 |
-
|
211 |
-
# load log's cfg
|
212 |
-
cfg_file = os.path.join(args.log_dir, 'cfg.yaml')
|
213 |
-
cfg = update_cfg(cfg_file)
|
214 |
-
|
215 |
-
if args.misc is not None:
|
216 |
-
cfg.merge_from_list(args.misc)
|
217 |
-
else:
|
218 |
-
parse_args(args)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|