mmpose/apis/inference_3d.py
def inference_interhand_3d_model(model,
                                 img_or_path,
                                 det_results,
                                 bbox_thr=None,
                                 format='xywh',
                                 dataset='InterHand3DDataset'):
    """Inference a single image with a list of hand bounding boxes.

    num_bboxes: N
    num_keypoints: K

    Args:
        model (nn.Module): The loaded pose model.
        img_or_path (str | np.ndarray): Image filename or loaded image.
        det_results (List[dict]): The 2D bbox sequences stored in a list.
            Each each element of the list is the bbox of one person, which
            contains:
                - "bbox" (ndarray[4 or 5]): The person bounding box,
                which contains 4 box coordinates (and score).
        dataset (str): Dataset name.
        format: bbox format ('xyxy' | 'xywh'). Default: 'xywh'.
            'xyxy' means (left, top, right, bottom),
            'xywh' means (left, top, width, height).

    Returns:
        List[dict]: 3D pose inference results. Each element is the result of
            an instance, which contains:
            - "keypoints_3d" (ndarray[K,3]): predicted 3D keypoints
            If there is no valid instance, an empty list will be returned.
    """

    assert format in ['xyxy', 'xywh']

    pose_results = []

    if len(det_results) == 0:
        return pose_results

    # Change for-loop preprocess each bbox to preprocess all bboxes at once.
    bboxes = np.array([box['bbox'] for box in det_results])

    # Select bboxes by score threshold
    if bbox_thr is not None:
        assert bboxes.shape[1] == 5
        valid_idx = np.where(bboxes[:, 4] > bbox_thr)[0]
        bboxes = bboxes[valid_idx]
        det_results = [det_results[i] for i in valid_idx]

    if format == 'xyxy':
        bboxes_xyxy = bboxes
        bboxes_xywh = _xyxy2xywh(bboxes)
    else:
        # format is already 'xywh'
        bboxes_xywh = bboxes
        bboxes_xyxy = _xywh2xyxy(bboxes)

    # if bbox_thr remove all bounding box
    if len(bboxes_xywh) == 0:
        return []

    cfg = model.cfg
    device = next(model.parameters()).device

    # build the data pipeline
    channel_order = cfg.test_pipeline[0].get('channel_order', 'rgb')
    test_pipeline = [LoadImage(channel_order=channel_order)
                     ] + cfg.test_pipeline[1:]
    test_pipeline = Compose(test_pipeline)

    assert len(bboxes[0]) in [4, 5]

    if dataset == 'InterHand3DDataset':
        flip_pairs = [[i, 21 + i] for i in range(21)]
    else:
        raise NotImplementedError()

    batch_data = []
    for bbox in bboxes:
        center, scale = _box2cs(cfg, bbox)

        # prepare data
        data = {
            'img_or_path':
            img_or_path,
            'center':
            center,
            'scale':
            scale,
            'bbox_score':
            bbox[4] if len(bbox) == 5 else 1,
            'bbox_id':
            0,  # need to be assigned if batch_size > 1
            'dataset':
            dataset,
            'joints_3d':
            np.zeros((cfg.data_cfg.num_joints, 3), dtype=np.float32),
            'joints_3d_visible':
            np.zeros((cfg.data_cfg.num_joints, 3), dtype=np.float32),
            'rotation':
            0,
            'ann_info': {
                'image_size': np.array(cfg.data_cfg['image_size']),
                'num_joints': cfg.data_cfg['num_joints'],
                'flip_pairs': flip_pairs,
                'heatmap3d_depth_bound': cfg.data_cfg['heatmap3d_depth_bound'],
                'heatmap_size_root': cfg.data_cfg['heatmap_size_root'],
                'root_depth_bound': cfg.data_cfg['root_depth_bound']
            }
        }

        data = test_pipeline(data)
        batch_data.append(data)

    batch_data = collate(batch_data, samples_per_gpu=1)

    if next(model.parameters()).is_cuda:
        # scatter not work so just move image to cuda device
        batch_data['img'] = batch_data['img'].to(device)
    # get all img_metas of each bounding box
    batch_data['img_metas'] = [
        img_metas[0] for img_metas in batch_data['img_metas'].data
    ]

    # forward the model
    with torch.no_grad():
        result = model(
            img=batch_data['img'],
            img_metas=batch_data['img_metas'],
            return_loss=False)

    poses_3d = result['preds']
    rel_root_depth = result['rel_root_depth']
    hand_type = result['hand_type']
    if poses_3d.shape[-1] != 4:
        assert poses_3d.shape[-1] == 3
        dummy_score = np.ones(
            poses_3d.shape[:-1] + (1, ), dtype=poses_3d.dtype)
        poses_3d = np.concatenate((poses_3d, dummy_score), axis=-1)

    # add relative root depth to left hand joints
    poses_3d[:, 21:, 2] += rel_root_depth

    # set joint scores according to hand type
    poses_3d[:, :21, 3] *= hand_type[:, [0]]
    poses_3d[:, 21:, 3] *= hand_type[:, [1]]

    pose_results = []
    for pose_3d, person_res, bbox_xyxy in zip(poses_3d, det_results,
                                              bboxes_xyxy):
        pose_res = person_res.copy()
        pose_res['keypoints_3d'] = pose_3d
        pose_res['bbox'] = bbox_xyxy
        pose_results.append(pose_res)

    return pose_results
Inference a single image with a list of hand bounding boxes.

    num_bboxes: N
    num_keypoints: K

    Args:
        model (nn.Module): The loaded pose model.
        img_or_path (str | np.ndarray): Image filename or loaded image.
        det_results (List[dict]): The 2D bbox sequences stored in a list.
            Each each element of the list is the bbox of one person, which
            contains:
                - "bbox" (ndarray[4 or 5]): The person bounding box,
                which contains 4 box coordinates (and score).
        dataset (str): Dataset name.
        format: bbox format ('xyxy' | 'xywh'). Default: 'xywh'.
            'xyxy' means (left, top, right, bottom),
            'xywh' means (left, top, width, height).

    Returns:
        List[dict]: 3D pose inference results. Each element is the result of
            an instance, which contains:
            - "keypoints_3d" (ndarray[K,3]): predicted 3D keypoints
            If there is no valid instance, an empty list will be returned.
    
Inference a single image with a list of hand bounding boxes.
Inference a single image with a list of hand bounding boxes .
def inference_interhand_3d_model(model,
                                 img_or_path,
                                 det_results,
                                 bbox_thr=None,
                                 format='xywh',
                                 dataset='InterHand3DDataset'):
    assert format in ['xyxy', 'xywh']
    pose_results = []
    if len(det_results) == 0:
        return pose_results
    bboxes = np.array([box['bbox'] for box in det_results])
    if bbox_thr is not None:
        assert bboxes.shape[1] == 5
        valid_idx = np.where(bboxes[:, 4] > bbox_thr)[0]
        bboxes = bboxes[valid_idx]
        det_results = [det_results[i] for i in valid_idx]
    if format == 'xyxy':
        bboxes_xyxy = bboxes
        bboxes_xywh = _xyxy2xywh(bboxes)
    else:
        bboxes_xywh = bboxes
        bboxes_xyxy = _xywh2xyxy(bboxes)
    if len(bboxes_xywh) == 0:
        return []
    cfg = model.cfg
    device = next(model.parameters()).device
    channel_order = cfg.test_pipeline[0].get('channel_order', 'rgb')
    test_pipeline = [LoadImage(channel_order=channel_order)
                     ] + cfg.test_pipeline[1:]
    test_pipeline = Compose(test_pipeline)
    assert len(bboxes[0]) in [4, 5]
    if dataset == 'InterHand3DDataset':
        flip_pairs = [[i, 21 + i] for i in range(21)]
    else:
        raise NotImplementedError()
    batch_data = []
    for bbox in bboxes:
        center, scale = _box2cs(cfg, bbox)
        data = {
            'img_or_path':
            img_or_path,
            'center':
            center,
            'scale':
            scale,
            'bbox_score':
            bbox[4] if len(bbox) == 5 else 1,
            'bbox_id':
            0,  
            'dataset':
            dataset,
            'joints_3d':
            np.zeros((cfg.data_cfg.num_joints, 3), dtype=np.float32),
            'joints_3d_visible':
            np.zeros((cfg.data_cfg.num_joints, 3), dtype=np.float32),
            'rotation':
            0,
            'ann_info': {
                'image_size': np.array(cfg.data_cfg['image_size']),
                'num_joints': cfg.data_cfg['num_joints'],
                'flip_pairs': flip_pairs,
                'heatmap3d_depth_bound': cfg.data_cfg['heatmap3d_depth_bound'],
                'heatmap_size_root': cfg.data_cfg['heatmap_size_root'],
                'root_depth_bound': cfg.data_cfg['root_depth_bound']
            }
        }
        data = test_pipeline(data)
        batch_data.append(data)
    batch_data = collate(batch_data, samples_per_gpu=1)
    if next(model.parameters()).is_cuda:
        batch_data['img'] = batch_data['img'].to(device)
    batch_data['img_metas'] = [
        img_metas[0] for img_metas in batch_data['img_metas'].data
    ]
    with torch.no_grad():
        result = model(
            img=batch_data['img'],
            img_metas=batch_data['img_metas'],
            return_loss=False)
    poses_3d = result['preds']
    rel_root_depth = result['rel_root_depth']
    hand_type = result['hand_type']
    if poses_3d.shape[-1] != 4:
        assert poses_3d.shape[-1] == 3
        dummy_score = np.ones(
            poses_3d.shape[:-1] + (1, ), dtype=poses_3d.dtype)
        poses_3d = np.concatenate((poses_3d, dummy_score), axis=-1)
    poses_3d[:, 21:, 2] += rel_root_depth
    poses_3d[:, :21, 3] *= hand_type[:, [0]]
    poses_3d[:, 21:, 3] *= hand_type[:, [1]]
    pose_results = []
    for pose_3d, person_res, bbox_xyxy in zip(poses_3d, det_results,
                                              bboxes_xyxy):
        pose_res = person_res.copy()
        pose_res['keypoints_3d'] = pose_3d
        pose_res['bbox'] = bbox_xyxy
        pose_results.append(pose_res)
    return pose_results
def inference_interhand_3d_model ( model , img_or_path , det_results , bbox_thr = None , format = 'xywh' , dataset = 'InterHand3DDataset' ) : assert format in [ 'xyxy' , 'xywh' ] pose_results = [ ] if len ( det_results ) == 0 : return pose_results bboxes = np . array ( [ box [ 'bbox' ] for box in det_results ] ) if bbox_thr is not None : assert bboxes . shape [ 1 ] == 5 valid_idx = np . where ( bboxes [ : , 4 ] > bbox_thr ) [ 0 ] bboxes = bboxes [ valid_idx ] det_results = [ det_results [ i ] for i in valid_idx ] if format == 'xyxy' : bboxes_xyxy = bboxes bboxes_xywh = _xyxy2xywh ( bboxes ) else : bboxes_xywh = bboxes bboxes_xyxy = _xywh2xyxy ( bboxes ) if len ( bboxes_xywh ) == 0 : return [ ] cfg = model . cfg device = next ( model . parameters ( ) ) . device channel_order = cfg . test_pipeline [ 0 ] . get ( 'channel_order' , 'rgb' ) test_pipeline = [ LoadImage ( channel_order = channel_order ) ] + cfg . test_pipeline [ 1 : ] test_pipeline = Compose ( test_pipeline ) assert len ( bboxes [ 0 ] ) in [ 4 , 5 ] if dataset == 'InterHand3DDataset' : flip_pairs = [ [ i , 21 + i ] for i in range ( 21 ) ] else : raise NotImplementedError ( ) batch_data = [ ] for bbox in bboxes : center , scale = _box2cs ( cfg , bbox ) data = { 'img_or_path' : img_or_path , 'center' : center , 'scale' : scale , 'bbox_score' : bbox [ 4 ] if len ( bbox ) == 5 else 1 , 'bbox_id' : 0 , 'dataset' : dataset , 'joints_3d' : np . zeros ( ( cfg . data_cfg . num_joints , 3 ) , dtype = np . float32 ) , 'joints_3d_visible' : np . zeros ( ( cfg . data_cfg . num_joints , 3 ) , dtype = np . float32 ) , 'rotation' : 0 , 'ann_info' : { 'image_size' : np . array ( cfg . data_cfg [ 'image_size' ] ) , 'num_joints' : cfg . data_cfg [ 'num_joints' ] , 'flip_pairs' : flip_pairs , 'heatmap3d_depth_bound' : cfg . data_cfg [ 'heatmap3d_depth_bound' ] , 'heatmap_size_root' : cfg . data_cfg [ 'heatmap_size_root' ] , 'root_depth_bound' : cfg . data_cfg [ 'root_depth_bound' ] } } data = test_pipeline ( data ) batch_data . append ( data ) batch_data = collate ( batch_data , samples_per_gpu = 1 ) if next ( model . parameters ( ) ) . is_cuda : batch_data [ 'img' ] = batch_data [ 'img' ] . to ( device ) batch_data [ 'img_metas' ] = [ img_metas [ 0 ] for img_metas in batch_data [ 'img_metas' ] . data ] with torch . no_grad ( ) : result = model ( img = batch_data [ 'img' ] , img_metas = batch_data [ 'img_metas' ] , return_loss = False ) poses_3d = result [ 'preds' ] rel_root_depth = result [ 'rel_root_depth' ] hand_type = result [ 'hand_type' ] if poses_3d . shape [ - 1 ] != 4 : assert poses_3d . shape [ - 1 ] == 3 dummy_score = np . ones ( poses_3d . shape [ : - 1 ] + ( 1 , ) , dtype = poses_3d . dtype ) poses_3d = np . concatenate ( ( poses_3d , dummy_score ) , axis = - 1 ) poses_3d [ : , 21 : , 2 ] += rel_root_depth poses_3d [ : , : 21 , 3 ] *= hand_type [ : , [ 0 ] ] poses_3d [ : , 21 : , 3 ] *= hand_type [ : , [ 1 ] ] pose_results = [ ] for pose_3d , person_res , bbox_xyxy in zip ( poses_3d , det_results , bboxes_xyxy ) : pose_res = person_res . copy ( ) pose_res [ 'keypoints_3d' ] = pose_3d pose_res [ 'bbox' ] = bbox_xyxy pose_results . append ( pose_res ) return pose_results
Inference a single image with a list of hand bounding boxes.
Inference a single image with a list of hand bounding boxes .
"""Inference a single image with a list of hand bounding boxes.

    num_bboxes: N
    num_keypoints: K

    Args:
        model (nn.Module): The loaded pose model.
        img_or_path (str | np.ndarray): Image filename or loaded image.
        det_results (List[dict]): The 2D bbox sequences stored in a list.
            Each each element of the list is the bbox of one person, which
            contains:
                - "bbox" (ndarray[4 or 5]): The person bounding box,
                which contains 4 box coordinates (and score).
        dataset (str): Dataset name.
        format: bbox format ('xyxy' | 'xywh'). Default: 'xywh'.
            'xyxy' means (left, top, right, bottom),
            'xywh' means (left, top, width, height).

    Returns:
        List[dict]: 3D pose inference results. Each element is the result of
            an instance, which contains:
            - "keypoints_3d" (ndarray[K,3]): predicted 3D keypoints
            If there is no valid instance, an empty list will be returned.
    """ # Change for-loop preprocess each bbox to preprocess all bboxes at once. # Select bboxes by score threshold # format is already 'xywh' # if bbox_thr remove all bounding box # build the data pipeline # prepare data # need to be assigned if batch_size > 1 # scatter not work so just move image to cuda device # get all img_metas of each bounding box # forward the model # add relative root depth to left hand joints # set joint scores according to hand type
3D pose inference results. Each element is the result of
an instance, which contains:
"keypoints_3d" (ndarray[K,3]): predicted 3D keypoints
If there is no valid instance, an empty list will be returned.
3D pose inference results . Each element is the result of an instance which contains : " keypoints_3d " ( ndarray [ K 3 ] ) : predicted 3D keypoints If there is no valid instance an empty list will be returned .
List[dict]
model
The loaded pose model.
The loaded pose model .
img_or_path
Image filename or loaded image.
Image filename or loaded image .
det_results
The 2D bbox sequences stored in a list.
Each each element of the list is the bbox of one person, which
contains:
"bbox" (ndarray[4 or 5]): The person bounding box,
which contains 4 box coordinates (and score).
The 2D bbox sequences stored in a list . Each each element of the list is the bbox of one person which contains : " bbox " ( ndarray [ 4 or 5 ] ) : The person bounding box which contains 4 box coordinates ( and score ) .
bbox_thr
format
bbox format ('xyxy' | 'xywh'). Default: 'xywh'.
'xyxy' means (left, top, right, bottom),
'xywh' means (left, top, width, height).
bbox format ( ' xyxy ' | ' xywh ' ) . Default : ' xywh ' . ' xyxy ' means ( left top right bottom ) ' xywh ' means ( left top width height ) .
dataset
None