def case_sample2(testres, filt_cfg, qaids=None, return_mask=False, verbose=None):
    "\n        Filters individual test result cases based on how they performed, what\n        tags they had, and various other things.\n\n        Args:\n            filt_cfg (dict):\n\n        Returns:\n            list: case_pos_list (list of (qx, cfgx)) or isvalid mask\n\n        CommandLine:\n            python -m wbia TestResult.case_sample2\n            python -m wbia TestResult.case_sample2:0\n            python -m wbia TestResult.case_sample2:1 --db GZ_ALL --filt :min_tags=1\n            python -m wbia TestResult.case_sample2:1 --db PZ_Master1 --filt :min_gf_tags=1\n\n            python -m wbia TestResult.case_sample2:2 --db PZ_Master1\n\n        Example:\n            >>> # DISABLE_DOCTEST\n            >>> # The same results is achievable with different filter config settings\n            >>> from wbia.expt.test_result import *  # NOQA\n            >>> from wbia.init import main_helpers\n            >>> verbose = True\n            >>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'])\n            >>> filt_cfg1 = {'fail': True}\n            >>> case_pos_list1 = testres.case_sample2(filt_cfg1)\n            >>> filt_cfg2 = {'min_gtrank': 1}\n            >>> case_pos_list2 = testres.case_sample2(filt_cfg2)\n            >>> filt_cfg3 = {'min_gtrank': 0}\n            >>> case_pos_list3 = testres.case_sample2(filt_cfg3)\n            >>> filt_cfg4 = {}\n            >>> case_pos_list4 = testres.case_sample2(filt_cfg4)\n            >>> assert np.all(case_pos_list1 == case_pos_list2), 'should be equiv configs'\n            >>> assert np.any(case_pos_list2 != case_pos_list3), 'should be diff configs'\n            >>> assert np.all(case_pos_list3 == case_pos_list4), 'should be equiv configs'\n            >>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'], t=['default:sv_on=[True,False]'])\n            >>> filt_cfg5 = filt_cfg1.copy()\n            >>> mask5 = testres.case_sample2(filt_cfg5, return_mask=True)\n            >>> case_pos_list5 = testres.case_sample2(filt_cfg5, return_mask=False)\n            >>> assert len(mask5.shape) == 2\n            >>> assert np.all(mask5.T[0] == mask5.T[1])\n            >>> filt_cfg6 = {'fail': True, 'allcfg': True}\n            >>> mask6 = testres.case_sample2(filt_cfg6, return_mask=True)\n            >>> assert np.all(mask6.T[0] == mask6.T[1])\n            >>> print(mask5)\n            >>> print(case_pos_list5)\n            >>> filt_cfg = filt_cfg7 = {'disagree': True}\n            >>> case_pos_list7 = testres.case_sample2(filt_cfg7, verbose=verbose)\n            >>> print(case_pos_list7)\n\n        Example:\n            >>> # SCRIPT\n            >>> from wbia.expt.test_result import *  # NOQA\n            >>> from wbia.init import main_helpers\n            >>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'])\n            >>> filt_cfg = main_helpers.testdata_filtcfg()\n            >>> case_pos_list = testres.case_sample2(filt_cfg)\n            >>> result = ('case_pos_list = %s' % (str(case_pos_list),))\n            >>> print(result)\n            >>> # Extra stuff\n            >>> all_tags = testres.get_all_tags()\n            >>> selcted_tags = ut.take(all_tags, case_pos_list.T[0])\n            >>> print('selcted_tags = %r' % (selcted_tags,))\n\n        Example:\n            >>> # DISABLE_DOCTEST\n            >>> from wbia.expt.test_result import *  # NOQA\n            >>> from wbia.init import main_helpers\n            >>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'], t=['default:K=[1,2,3]'])\n            >>> ut.exec_funckw(testres.case_sample2, globals())\n            >>> filt_cfg = {'fail': True, 'min_gtrank': 1, 'max_gtrank': None, 'min_gf_timedelta': '24h'}\n            >>> ibs, testres = main_helpers.testdata_expts('humpbacks_fb', a=['default:has_any=hasnotch,mingt=2,qindex=0:300,dindex=0:300'], t=['default:proot=BC_DTW,decision=max,crop_dim_size=500,crop_enabled=True,manual_extract=False,use_te_scorer=True,ignore_notch=True,te_net=annot_simple', 'default:proot=vsmany'], qaid_override=[12])\n            >>> filt_cfg = ':disagree=True,index=0:8,min_gtscore=.00001,require_all_cfg=True'\n            >>> #filt_cfg = cfghelpers.parse_argv_cfg('--filt')[0]\n            >>> case_pos_list = testres.case_sample2(filt_cfg, verbose=True)\n            >>> result = ('case_pos_list = %s' % (str(case_pos_list),))\n            >>> print(result)\n            >>> # Extra stuff\n            >>> all_tags = testres.get_all_tags()\n            >>> selcted_tags = ut.take(all_tags, case_pos_list.T[0])\n            >>> print('selcted_tags = %r' % (selcted_tags,))\n\n\n            logger.info('qaid = %r' % (qaid,))\n            logger.info('qx = %r' % (qx,))\n            logger.info('cfgxs = %r' % (cfgxs,))\n            # print testres info about this item\n            take_cfgs = ut.partial(ut.take, index_list=cfgxs)\n            take_qx = ut.partial(ut.take, index_list=qx)\n            truth_cfgs = ut.hmap_vals(take_qx, truth2_prop)\n            truth_item = ut.hmap_vals(take_cfgs, truth_cfgs, max_depth=1)\n            prop_cfgs = ut.hmap_vals(take_qx, prop2_mat)\n            prop_item = ut.hmap_vals(take_cfgs, prop_cfgs, max_depth=0)\n            logger.info('truth2_prop[item] = ' + ut.repr3(truth_item, nl=2))\n            logger.info('prop2_mat[item] = ' + ut.repr3(prop_item, nl=1))\n        "
    from wbia.expt import cfghelpers
    if (verbose is None):
        verbose = ut.NOT_QUIET
    if verbose:
        logger.info('[testres] case_sample2')
    if isinstance(filt_cfg, str):
        filt_cfg = [filt_cfg]
    if isinstance(filt_cfg, list):
        _combos = cfghelpers.parse_cfgstr_list2(filt_cfg, strict=False)
        filt_cfg = ut.flatten(_combos)[0]
    if isinstance(filt_cfg, str):
        _combos = cfghelpers.parse_cfgstr_list2([filt_cfg], strict=False)
        filt_cfg = ut.flatten(_combos)[0]
    if (filt_cfg is None):
        filt_cfg = {}
    qaids = (testres.get_test_qaids() if (qaids is None) else qaids)
    (truth2_prop, prop2_mat) = testres.get_truth2_prop(qaids)
    ibs = testres.ibs
    participates = prop2_mat['participates']
    is_valid = participates.copy()

    def unflat_tag_filterflags(tags_list, **kwargs):
        from wbia import tag_funcs
        (flat_tags, cumsum) = ut.invertible_flatten2(tags_list)
        flat_flags = tag_funcs.filterflags_general_tags(flat_tags, **kwargs)
        flags = np.array(ut.unflatten2(flat_flags, cumsum))
        return flags
    UTFF = unflat_tag_filterflags

    def cols_disagree(mat, val):
        "\n            is_success = prop2_mat['is_success']\n            "
        nCols = mat.shape[1]
        sums = mat.sum(axis=1)
        disagree_flags1d = np.logical_and((sums > 0), (sums < nCols))
        disagree_flags2d = np.tile(disagree_flags1d[:, None], (1, nCols))
        if (not val):
            flags = np.logical_not(disagree_flags2d)
        else:
            flags = disagree_flags2d
        return flags

    def cfg_scoresep(mat, val, op):
        "\n            Compares scores between different configs\n\n            op = operator.ge\n            is_success = prop2_mat['is_success']\n            "
        nCols = mat.shape[1]
        pdistx = vt.pdist_indicies(nCols)
        pdist_list = np.array([vt.safe_pdist(row) for row in mat])
        flags_list = op(pdist_list, val)
        colx_list = [np.unique(ut.flatten(ut.compress(pdistx, flags))) for flags in flags_list]
        offsets = np.arange(0, (nCols * len(mat)), step=nCols)
        idx_list = ut.flatten([(colx + offset) for (colx, offset) in zip(colx_list, offsets)])
        mask = vt.index_to_boolmask(idx_list, maxval=(offsets[(- 1)] + nCols))
        flags = mask.reshape(mat.shape)
        return flags
    rule_list = [('disagree', (lambda val: cols_disagree(prop2_mat['is_failure'], val))), ('min_gt_cfg_scoresep', (lambda val: cfg_scoresep(truth2_prop['gt']['score'], val, operator.ge))), ('fail', prop2_mat['is_failure']), ('success', prop2_mat['is_success']), ('min_gtrank', partial(operator.ge, truth2_prop['gt']['rank'])), ('max_gtrank', partial(operator.le, truth2_prop['gt']['rank'])), ('max_gtscore', partial(operator.le, truth2_prop['gt']['score'])), ('min_gtscore', partial(operator.ge, truth2_prop['gt']['score'])), ('min_gf_timedelta', partial(operator.ge, truth2_prop['gf']['timedelta'])), ('max_gf_timedelta', partial(operator.le, truth2_prop['gf']['timedelta'])), ('min_tags', (lambda val: UTFF(testres.get_all_tags(), min_num=val))), ('max_tags', (lambda val: UTFF(testres.get_all_tags(), max_num=val))), ('min_gf_tags', (lambda val: UTFF(testres.get_gf_tags(), min_num=val))), ('max_gf_tags', (lambda val: UTFF(testres.get_gf_tags(), max_num=val))), ('min_gt_tags', (lambda val: UTFF(testres.get_gt_tags(), min_num=val))), ('max_gt_tags', (lambda val: UTFF(testres.get_gt_tags(), max_num=val))), ('min_query_annot_tags', (lambda val: UTFF(testres.get_query_annot_tags(), min_num=val))), ('min_gt_annot_tags', (lambda val: UTFF(testres.get_gt_annot_tags(), min_num=val))), ('min_gtq_tags', (lambda val: UTFF(testres.get_gtquery_annot_tags(), min_num=val))), ('max_gtq_tags', (lambda val: UTFF(testres.get_gtquery_annot_tags(), max_num=val))), ('without_gf_tag', (lambda val: UTFF(testres.get_gf_tags(), has_none=val))), ('without_gt_tag', (lambda val: UTFF(testres.get_gt_tags(), has_none=val))), ('with_gf_tag', (lambda val: UTFF(testres.get_gf_tags(), has_any=val))), ('with_gt_tag', (lambda val: UTFF(testres.get_gt_tags(), has_any=val))), ('with_tag', (lambda val: UTFF(testres.get_all_tags(), has_any=val))), ('without_tag', (lambda val: UTFF(testres.get_all_tags(), has_none=val)))]
    rule_dict = ut.odict(rule_list)
    rule_list.append(('max_gf_td', rule_dict['max_gf_timedelta']))
    rule_list.append(('min_gf_td', rule_dict['min_gf_timedelta']))
    filt_cfg_ = copy.deepcopy(filt_cfg)
    for tdkey in filt_cfg_.keys():
        if tdkey.endswith('_timedelta'):
            filt_cfg_[tdkey] = ut.ensure_timedelta(filt_cfg_[tdkey])

    class VerbFilterInfo(object):

        def __init__(self):
            self.prev_num_valid = None

        def print_pre(self, is_valid, filt_cfg_):
            num_valid = is_valid.sum()
            logger.info(('[testres] Sampling from is_valid.size=%r with filt=%r' % (is_valid.size, ut.get_cfg_lbl(filt_cfg_))))
            logger.info(('  * is_valid.shape = %r' % (is_valid.shape,)))
            logger.info(('  * num_valid = %r' % (num_valid,)))
            self.prev_num_valid = num_valid

        def print_post(self, is_valid, flags, msg):
            if (flags is not None):
                num_passed = flags.sum()
            num_valid = is_valid.sum()
            num_invalidated = (self.prev_num_valid - num_valid)
            logger.info(msg)
            if (num_invalidated == 0):
                if (flags is not None):
                    logger.info(('  * num_passed = %r' % (num_passed,)))
                logger.info(('  * num_invalided = %r' % (num_invalidated,)))
            else:
                logger.info(('  * prev_num_valid = %r' % (self.prev_num_valid,)))
                logger.info(('  * num_valid = %r' % (num_valid,)))
            self.prev_num_valid = num_valid
    verbinfo = VerbFilterInfo()
    if verbose:
        verbinfo.print_pre(is_valid, filt_cfg_)
    ut.delete_keys(filt_cfg_, ['_cfgstr', '_cfgindex', '_cfgname', '_cfgtype'])
    valid_rules = []

    def poprule(rulename, default):
        valid_rules.append(rulename)
        return filt_cfg_.pop(rulename, default)
    allcfg = poprule('allcfg', None)
    orderby = poprule('orderby', None)
    reverse = poprule('reverse', None)
    sortasc = poprule('sortasc', None)
    sortdsc = poprule('sortdsc', poprule('sortdesc', None))
    max_pername = poprule('max_pername', None)
    require_all_cfg = poprule('require_all_cfg', None)
    index = poprule('index', None)
    rule_value_list = [poprule(key, None) for (key, rule) in rule_list]
    if (len(filt_cfg_) > 0):
        logger.info('ERROR')
        logger.info(('filtcfg valid rules are = %s' % (ut.repr2(valid_rules, nl=1),)))
        for key in filt_cfg_.keys():
            logger.info(('did you mean %r instead of %r?' % (ut.closet_words(key, valid_rules)[0], key)))
        raise NotImplementedError(('Unhandled filt_cfg.keys() = %r' % filt_cfg_.keys()))
    chosen_rule_idxs = ut.where([(val is not None) for val in rule_value_list])
    chosen_rules = ut.take(rule_list, chosen_rule_idxs)
    chosen_vals = ut.take(rule_value_list, chosen_rule_idxs)
    for ((key, rule), val) in zip(chosen_rules, chosen_vals):
        if isinstance(rule, np.ndarray):
            flags = (rule == val)
        else:
            flags = rule(val)
        flags = np.logical_and(flags, participates)
        is_valid = np.logical_and(is_valid, flags)
        if verbose:
            verbinfo.print_post(is_valid, flags, ('SampleRule: %s = %r' % (key, val)))
    if allcfg:
        is_valid = np.logical_or(np.logical_or.reduce(is_valid.T)[:, None], is_valid)
        is_valid = np.logical_and(is_valid, participates)
    (qx_list, cfgx_list) = np.nonzero(is_valid)
    if (sortdsc is not None):
        assert (orderby is None), 'use orderby or sortasc'
        assert (reverse is None), 'reverse does not work with sortdsc'
        orderby = sortdsc
        reverse = True
    elif (sortasc is not None):
        assert (reverse is None), 'reverse does not work with sortasc'
        assert (orderby is None), 'use orderby or sortasc'
        orderby = sortasc
        reverse = False
    else:
        reverse = False
    if (orderby is not None):
        import re
        order_values = None
        for prefix_pattern in ['^gt_?', '^gf_?']:
            prefix_match = re.match(prefix_pattern, orderby)
            if (prefix_match is not None):
                truth = prefix_pattern[1:3]
                propname = orderby[prefix_match.end():]
                if verbose:
                    logger.info(('Ordering by truth=%s propname=%s' % (truth, propname)))
                order_values = truth2_prop[truth][propname]
                break
        if (order_values is None):
            raise NotImplementedError(('Unknown orerby=%r' % (orderby,)))
    else:
        order_values = np.arange(is_valid.size).reshape(is_valid.shape)
    flat_order = order_values[is_valid]
    if verbose:
        if verbose:
            logger.info('Reversing ordering (descending)')
        else:
            logger.info('Normal ordering (ascending)')
    if reverse:
        sortx = flat_order.argsort()[::(- 1)]
    else:
        sortx = flat_order.argsort()
    qx_list = qx_list.take(sortx, axis=0)
    cfgx_list = cfgx_list.take(sortx, axis=0)
    if (max_pername is not None):
        if verbose:
            logger.info(('Returning at most %d cases per name ' % (max_pername,)))
        _qaid_list = np.take(qaids, qx_list)
        _qnid_list = ibs.get_annot_nids(_qaid_list)
        _valid_idxs = []
        seen_ = ut.ddict((lambda : 0))
        for (idx, _qnid) in enumerate(_qnid_list):
            if (seen_[_qnid] < max_pername):
                seen_[_qnid] += 1
                _valid_idxs.append(idx)
        _qx_list = qx_list[_valid_idxs]
        _cfgx_list = cfgx_list[_valid_idxs]
        _valid_index = np.vstack((_qx_list, _cfgx_list)).T
        is_valid = vt.index_to_boolmask(_valid_index, is_valid.shape, isflat=False)
        qx_list = _qx_list
        cfgx_list = _cfgx_list
    if require_all_cfg:
        if verbose:
            prev_num_valid = is_valid.sum()
            logger.info('Enforcing that all configs must pass filters')
            logger.info(('  * prev_num_valid = %r' % (prev_num_valid,)))
        qx2_valid_cfgs = ut.group_items(cfgx_list, qx_list)
        hasall_cfg = [(len(qx2_valid_cfgs[qx]) == testres.nConfig) for qx in qx_list]
        _qx_list = qx_list.compress(hasall_cfg)
        _cfgx_list = cfgx_list.compress(hasall_cfg)
        _valid_index = np.vstack((_qx_list, _cfgx_list)).T
        is_valid = vt.index_to_boolmask(_valid_index, is_valid.shape, isflat=False)
        qx_list = _qx_list
        cfgx_list = _cfgx_list
        if verbose:
            verbinfo.print_post(is_valid, None, 'Enforcing that all configs must pass filters')
    if (index is not None):
        if isinstance(index, str):
            index = ut.smart_cast(index, slice)
        _qx_list = ut.take(qx_list, index)
        _cfgx_list = ut.take(cfgx_list, index)
        _valid_index = np.vstack((_qx_list, _cfgx_list)).T
        is_valid = vt.index_to_boolmask(_valid_index, is_valid.shape, isflat=False)
        qx_list = _qx_list
        cfgx_list = _cfgx_list
        if verbose:
            verbinfo.print_post(is_valid, None, ('Taking index=%r sample from len(qx_list) = %r' % (index, len(qx_list))))
    if (not return_mask):
        case_pos_list = np.vstack((qx_list, cfgx_list)).T
        case_identifier = case_pos_list
    else:
        if verbose:
            logger.info('Converting cases indicies to a 2d-mask')
        case_identifier = is_valid
    if verbose:
        logger.info('Finished case filtering')
        logger.info('Final case stats:')
        qx_hist = ut.dict_hist(qx_list)
        logger.info(('config per query stats: %r' % (ut.get_stats_str(qx_hist.values()),)))
        logger.info(('query per config stats: %r' % (ut.get_stats_str(ut.dict_hist(cfgx_list).values()),)))
    return case_identifier
Filters individual test result cases based on how they performed, what
tags they had, and various other things.

Args:
    filt_cfg (dict):

Returns:
    list: case_pos_list (list of (qx, cfgx)) or isvalid mask

CommandLine:
    python -m wbia TestResult.case_sample2
    python -m wbia TestResult.case_sample2:0
    python -m wbia TestResult.case_sample2:1 --db GZ_ALL --filt :min_tags=1
    python -m wbia TestResult.case_sample2:1 --db PZ_Master1 --filt :min_gf_tags=1

    python -m wbia TestResult.case_sample2:2 --db PZ_Master1

Example:
    >>> # DISABLE_DOCTEST
    >>> # The same results is achievable with different filter config settings
    >>> from wbia.expt.test_result import *  # NOQA
    >>> from wbia.init import main_helpers
    >>> verbose = True
    >>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'])
    >>> filt_cfg1 = {'fail': True}
    >>> case_pos_list1 = testres.case_sample2(filt_cfg1)
    >>> filt_cfg2 = {'min_gtrank': 1}
    >>> case_pos_list2 = testres.case_sample2(filt_cfg2)
    >>> filt_cfg3 = {'min_gtrank': 0}
    >>> case_pos_list3 = testres.case_sample2(filt_cfg3)
    >>> filt_cfg4 = {}
    >>> case_pos_list4 = testres.case_sample2(filt_cfg4)
    >>> assert np.all(case_pos_list1 == case_pos_list2), 'should be equiv configs'
    >>> assert np.any(case_pos_list2 != case_pos_list3), 'should be diff configs'
    >>> assert np.all(case_pos_list3 == case_pos_list4), 'should be equiv configs'
    >>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'], t=['default:sv_on=[True,False]'])
    >>> filt_cfg5 = filt_cfg1.copy()
    >>> mask5 = testres.case_sample2(filt_cfg5, return_mask=True)
    >>> case_pos_list5 = testres.case_sample2(filt_cfg5, return_mask=False)
    >>> assert len(mask5.shape) == 2
    >>> assert np.all(mask5.T[0] == mask5.T[1])
    >>> filt_cfg6 = {'fail': True, 'allcfg': True}
    >>> mask6 = testres.case_sample2(filt_cfg6, return_mask=True)
    >>> assert np.all(mask6.T[0] == mask6.T[1])
    >>> print(mask5)
    >>> print(case_pos_list5)
    >>> filt_cfg = filt_cfg7 = {'disagree': True}
    >>> case_pos_list7 = testres.case_sample2(filt_cfg7, verbose=verbose)
    >>> print(case_pos_list7)

Example:
    >>> # SCRIPT
    >>> from wbia.expt.test_result import *  # NOQA
    >>> from wbia.init import main_helpers
    >>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'])
    >>> filt_cfg = main_helpers.testdata_filtcfg()
    >>> case_pos_list = testres.case_sample2(filt_cfg)
    >>> result = ('case_pos_list = %s' % (str(case_pos_list),))
    >>> print(result)
    >>> # Extra stuff
    >>> all_tags = testres.get_all_tags()
    >>> selcted_tags = ut.take(all_tags, case_pos_list.T[0])
    >>> print('selcted_tags = %r' % (selcted_tags,))

Example:
    >>> # DISABLE_DOCTEST
    >>> from wbia.expt.test_result import *  # NOQA
    >>> from wbia.init import main_helpers
    >>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'], t=['default:K=[1,2,3]'])
    >>> ut.exec_funckw(testres.case_sample2, globals())
    >>> filt_cfg = {'fail': True, 'min_gtrank': 1, 'max_gtrank': None, 'min_gf_timedelta': '24h'}
    >>> ibs, testres = main_helpers.testdata_expts('humpbacks_fb', a=['default:has_any=hasnotch,mingt=2,qindex=0:300,dindex=0:300'], t=['default:proot=BC_DTW,decision=max,crop_dim_size=500,crop_enabled=True,manual_extract=False,use_te_scorer=True,ignore_notch=True,te_net=annot_simple', 'default:proot=vsmany'], qaid_override=[12])
    >>> filt_cfg = ':disagree=True,index=0:8,min_gtscore=.00001,require_all_cfg=True'
    >>> #filt_cfg = cfghelpers.parse_argv_cfg('--filt')[0]
    >>> case_pos_list = testres.case_sample2(filt_cfg, verbose=True)
    >>> result = ('case_pos_list = %s' % (str(case_pos_list),))
    >>> print(result)
    >>> # Extra stuff
    >>> all_tags = testres.get_all_tags()
    >>> selcted_tags = ut.take(all_tags, case_pos_list.T[0])
    >>> print('selcted_tags = %r' % (selcted_tags,))


    logger.info('qaid = %r' % (qaid,))
    logger.info('qx = %r' % (qx,))
    logger.info('cfgxs = %r' % (cfgxs,))
    # print testres info about this item
    take_cfgs = ut.partial(ut.take, index_list=cfgxs)
    take_qx = ut.partial(ut.take, index_list=qx)
    truth_cfgs = ut.hmap_vals(take_qx, truth2_prop)
    truth_item = ut.hmap_vals(take_cfgs, truth_cfgs, max_depth=1)
    prop_cfgs = ut.hmap_vals(take_qx, prop2_mat)
    prop_item = ut.hmap_vals(take_cfgs, prop_cfgs, max_depth=0)
    logger.info('truth2_prop[item] = ' + ut.repr3(truth_item, nl=2))
    logger.info('prop2_mat[item] = ' + ut.repr3(prop_item, nl=1))
def case_sample2(testres, filt_cfg, qaids=None, return_mask=False, verbose=None):
    "\n        Filters individual test result cases based on how they performed, what\n        tags they had, and various other things.\n\n        Args:\n            filt_cfg (dict):\n\n        Returns:\n            list: case_pos_list (list of (qx, cfgx)) or isvalid mask\n\n        CommandLine:\n            python -m wbia TestResult.case_sample2\n            python -m wbia TestResult.case_sample2:0\n            python -m wbia TestResult.case_sample2:1 --db GZ_ALL --filt :min_tags=1\n            python -m wbia TestResult.case_sample2:1 --db PZ_Master1 --filt :min_gf_tags=1\n\n            python -m wbia TestResult.case_sample2:2 --db PZ_Master1\n\n        Example:\n            >>> # DISABLE_DOCTEST\n            >>> # The same results is achievable with different filter config settings\n            >>> from wbia.expt.test_result import *  # NOQA\n            >>> from wbia.init import main_helpers\n            >>> verbose = True\n            >>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'])\n            >>> filt_cfg1 = {'fail': True}\n            >>> case_pos_list1 = testres.case_sample2(filt_cfg1)\n            >>> filt_cfg2 = {'min_gtrank': 1}\n            >>> case_pos_list2 = testres.case_sample2(filt_cfg2)\n            >>> filt_cfg3 = {'min_gtrank': 0}\n            >>> case_pos_list3 = testres.case_sample2(filt_cfg3)\n            >>> filt_cfg4 = {}\n            >>> case_pos_list4 = testres.case_sample2(filt_cfg4)\n            >>> assert np.all(case_pos_list1 == case_pos_list2), 'should be equiv configs'\n            >>> assert np.any(case_pos_list2 != case_pos_list3), 'should be diff configs'\n            >>> assert np.all(case_pos_list3 == case_pos_list4), 'should be equiv configs'\n            >>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'], t=['default:sv_on=[True,False]'])\n            >>> filt_cfg5 = filt_cfg1.copy()\n            >>> mask5 = testres.case_sample2(filt_cfg5, return_mask=True)\n            >>> case_pos_list5 = testres.case_sample2(filt_cfg5, return_mask=False)\n            >>> assert len(mask5.shape) == 2\n            >>> assert np.all(mask5.T[0] == mask5.T[1])\n            >>> filt_cfg6 = {'fail': True, 'allcfg': True}\n            >>> mask6 = testres.case_sample2(filt_cfg6, return_mask=True)\n            >>> assert np.all(mask6.T[0] == mask6.T[1])\n            >>> print(mask5)\n            >>> print(case_pos_list5)\n            >>> filt_cfg = filt_cfg7 = {'disagree': True}\n            >>> case_pos_list7 = testres.case_sample2(filt_cfg7, verbose=verbose)\n            >>> print(case_pos_list7)\n\n        Example:\n            >>> # SCRIPT\n            >>> from wbia.expt.test_result import *  # NOQA\n            >>> from wbia.init import main_helpers\n            >>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'])\n            >>> filt_cfg = main_helpers.testdata_filtcfg()\n            >>> case_pos_list = testres.case_sample2(filt_cfg)\n            >>> result = ('case_pos_list = %s' % (str(case_pos_list),))\n            >>> print(result)\n            >>> # Extra stuff\n            >>> all_tags = testres.get_all_tags()\n            >>> selcted_tags = ut.take(all_tags, case_pos_list.T[0])\n            >>> print('selcted_tags = %r' % (selcted_tags,))\n\n        Example:\n            >>> # DISABLE_DOCTEST\n            >>> from wbia.expt.test_result import *  # NOQA\n            >>> from wbia.init import main_helpers\n            >>> ibs, testres = main_helpers.testdata_expts('PZ_MTEST', a=['ctrl'], t=['default:K=[1,2,3]'])\n            >>> ut.exec_funckw(testres.case_sample2, globals())\n            >>> filt_cfg = {'fail': True, 'min_gtrank': 1, 'max_gtrank': None, 'min_gf_timedelta': '24h'}\n            >>> ibs, testres = main_helpers.testdata_expts('humpbacks_fb', a=['default:has_any=hasnotch,mingt=2,qindex=0:300,dindex=0:300'], t=['default:proot=BC_DTW,decision=max,crop_dim_size=500,crop_enabled=True,manual_extract=False,use_te_scorer=True,ignore_notch=True,te_net=annot_simple', 'default:proot=vsmany'], qaid_override=[12])\n            >>> filt_cfg = ':disagree=True,index=0:8,min_gtscore=.00001,require_all_cfg=True'\n            >>> #filt_cfg = cfghelpers.parse_argv_cfg('--filt')[0]\n            >>> case_pos_list = testres.case_sample2(filt_cfg, verbose=True)\n            >>> result = ('case_pos_list = %s' % (str(case_pos_list),))\n            >>> print(result)\n            >>> # Extra stuff\n            >>> all_tags = testres.get_all_tags()\n            >>> selcted_tags = ut.take(all_tags, case_pos_list.T[0])\n            >>> print('selcted_tags = %r' % (selcted_tags,))\n\n\n            logger.info('qaid = %r' % (qaid,))\n            logger.info('qx = %r' % (qx,))\n            logger.info('cfgxs = %r' % (cfgxs,))\n            # print testres info about this item\n            take_cfgs = ut.partial(ut.take, index_list=cfgxs)\n            take_qx = ut.partial(ut.take, index_list=qx)\n            truth_cfgs = ut.hmap_vals(take_qx, truth2_prop)\n            truth_item = ut.hmap_vals(take_cfgs, truth_cfgs, max_depth=1)\n            prop_cfgs = ut.hmap_vals(take_qx, prop2_mat)\n            prop_item = ut.hmap_vals(take_cfgs, prop_cfgs, max_depth=0)\n            logger.info('truth2_prop[item] = ' + ut.repr3(truth_item, nl=2))\n            logger.info('prop2_mat[item] = ' + ut.repr3(prop_item, nl=1))\n        "
    from wbia.expt import cfghelpers
    if (verbose is None):
        verbose = ut.NOT_QUIET
    if verbose:
        logger.info('[testres] case_sample2')
    if isinstance(filt_cfg, str):
        filt_cfg = [filt_cfg]
    if isinstance(filt_cfg, list):
        _combos = cfghelpers.parse_cfgstr_list2(filt_cfg, strict=False)
        filt_cfg = ut.flatten(_combos)[0]
    if isinstance(filt_cfg, str):
        _combos = cfghelpers.parse_cfgstr_list2([filt_cfg], strict=False)
        filt_cfg = ut.flatten(_combos)[0]
    if (filt_cfg is None):
        filt_cfg = {}
    qaids = (testres.get_test_qaids() if (qaids is None) else qaids)
    (truth2_prop, prop2_mat) = testres.get_truth2_prop(qaids)
    ibs = testres.ibs
    participates = prop2_mat['participates']
    is_valid = participates.copy()

    def unflat_tag_filterflags(tags_list, **kwargs):
        from wbia import tag_funcs
        (flat_tags, cumsum) = ut.invertible_flatten2(tags_list)
        flat_flags = tag_funcs.filterflags_general_tags(flat_tags, **kwargs)
        flags = np.array(ut.unflatten2(flat_flags, cumsum))
        return flags
    UTFF = unflat_tag_filterflags

    def cols_disagree(mat, val):
        "\n            is_success = prop2_mat['is_success']\n            "
        nCols = mat.shape[1]
        sums = mat.sum(axis=1)
        disagree_flags1d = np.logical_and((sums > 0), (sums < nCols))
        disagree_flags2d = np.tile(disagree_flags1d[:, None], (1, nCols))
        if (not val):
            flags = np.logical_not(disagree_flags2d)
        else:
            flags = disagree_flags2d
        return flags

    def cfg_scoresep(mat, val, op):
        "\n            Compares scores between different configs\n\n            op = operator.ge\n            is_success = prop2_mat['is_success']\n            "
        nCols = mat.shape[1]
        pdistx = vt.pdist_indicies(nCols)
        pdist_list = np.array([vt.safe_pdist(row) for row in mat])
        flags_list = op(pdist_list, val)
        colx_list = [np.unique(ut.flatten(ut.compress(pdistx, flags))) for flags in flags_list]
        offsets = np.arange(0, (nCols * len(mat)), step=nCols)
        idx_list = ut.flatten([(colx + offset) for (colx, offset) in zip(colx_list, offsets)])
        mask = vt.index_to_boolmask(idx_list, maxval=(offsets[(- 1)] + nCols))
        flags = mask.reshape(mat.shape)
        return flags
    rule_list = [('disagree', (lambda val: cols_disagree(prop2_mat['is_failure'], val))), ('min_gt_cfg_scoresep', (lambda val: cfg_scoresep(truth2_prop['gt']['score'], val, operator.ge))), ('fail', prop2_mat['is_failure']), ('success', prop2_mat['is_success']), ('min_gtrank', partial(operator.ge, truth2_prop['gt']['rank'])), ('max_gtrank', partial(operator.le, truth2_prop['gt']['rank'])), ('max_gtscore', partial(operator.le, truth2_prop['gt']['score'])), ('min_gtscore', partial(operator.ge, truth2_prop['gt']['score'])), ('min_gf_timedelta', partial(operator.ge, truth2_prop['gf']['timedelta'])), ('max_gf_timedelta', partial(operator.le, truth2_prop['gf']['timedelta'])), ('min_tags', (lambda val: UTFF(testres.get_all_tags(), min_num=val))), ('max_tags', (lambda val: UTFF(testres.get_all_tags(), max_num=val))), ('min_gf_tags', (lambda val: UTFF(testres.get_gf_tags(), min_num=val))), ('max_gf_tags', (lambda val: UTFF(testres.get_gf_tags(), max_num=val))), ('min_gt_tags', (lambda val: UTFF(testres.get_gt_tags(), min_num=val))), ('max_gt_tags', (lambda val: UTFF(testres.get_gt_tags(), max_num=val))), ('min_query_annot_tags', (lambda val: UTFF(testres.get_query_annot_tags(), min_num=val))), ('min_gt_annot_tags', (lambda val: UTFF(testres.get_gt_annot_tags(), min_num=val))), ('min_gtq_tags', (lambda val: UTFF(testres.get_gtquery_annot_tags(), min_num=val))), ('max_gtq_tags', (lambda val: UTFF(testres.get_gtquery_annot_tags(), max_num=val))), ('without_gf_tag', (lambda val: UTFF(testres.get_gf_tags(), has_none=val))), ('without_gt_tag', (lambda val: UTFF(testres.get_gt_tags(), has_none=val))), ('with_gf_tag', (lambda val: UTFF(testres.get_gf_tags(), has_any=val))), ('with_gt_tag', (lambda val: UTFF(testres.get_gt_tags(), has_any=val))), ('with_tag', (lambda val: UTFF(testres.get_all_tags(), has_any=val))), ('without_tag', (lambda val: UTFF(testres.get_all_tags(), has_none=val)))]
    rule_dict = ut.odict(rule_list)
    rule_list.append(('max_gf_td', rule_dict['max_gf_timedelta']))
    rule_list.append(('min_gf_td', rule_dict['min_gf_timedelta']))
    filt_cfg_ = copy.deepcopy(filt_cfg)
    for tdkey in filt_cfg_.keys():
        if tdkey.endswith('_timedelta'):
            filt_cfg_[tdkey] = ut.ensure_timedelta(filt_cfg_[tdkey])

    class VerbFilterInfo(object):

        def __init__(self):
            self.prev_num_valid = None

        def print_pre(self, is_valid, filt_cfg_):
            num_valid = is_valid.sum()
            logger.info(('[testres] Sampling from is_valid.size=%r with filt=%r' % (is_valid.size, ut.get_cfg_lbl(filt_cfg_))))
            logger.info(('  * is_valid.shape = %r' % (is_valid.shape,)))
            logger.info(('  * num_valid = %r' % (num_valid,)))
            self.prev_num_valid = num_valid

        def print_post(self, is_valid, flags, msg):
            if (flags is not None):
                num_passed = flags.sum()
            num_valid = is_valid.sum()
            num_invalidated = (self.prev_num_valid - num_valid)
            logger.info(msg)
            if (num_invalidated == 0):
                if (flags is not None):
                    logger.info(('  * num_passed = %r' % (num_passed,)))
                logger.info(('  * num_invalided = %r' % (num_invalidated,)))
            else:
                logger.info(('  * prev_num_valid = %r' % (self.prev_num_valid,)))
                logger.info(('  * num_valid = %r' % (num_valid,)))
            self.prev_num_valid = num_valid
    verbinfo = VerbFilterInfo()
    if verbose:
        verbinfo.print_pre(is_valid, filt_cfg_)
    ut.delete_keys(filt_cfg_, ['_cfgstr', '_cfgindex', '_cfgname', '_cfgtype'])
    valid_rules = []

    def poprule(rulename, default):
        valid_rules.append(rulename)
        return filt_cfg_.pop(rulename, default)
    allcfg = poprule('allcfg', None)
    orderby = poprule('orderby', None)
    reverse = poprule('reverse', None)
    sortasc = poprule('sortasc', None)
    sortdsc = poprule('sortdsc', poprule('sortdesc', None))
    max_pername = poprule('max_pername', None)
    require_all_cfg = poprule('require_all_cfg', None)
    index = poprule('index', None)
    rule_value_list = [poprule(key, None) for (key, rule) in rule_list]
    if (len(filt_cfg_) > 0):
        logger.info('ERROR')
        logger.info(('filtcfg valid rules are = %s' % (ut.repr2(valid_rules, nl=1),)))
        for key in filt_cfg_.keys():
            logger.info(('did you mean %r instead of %r?' % (ut.closet_words(key, valid_rules)[0], key)))
        raise NotImplementedError(('Unhandled filt_cfg.keys() = %r' % filt_cfg_.keys()))
    chosen_rule_idxs = ut.where([(val is not None) for val in rule_value_list])
    chosen_rules = ut.take(rule_list, chosen_rule_idxs)
    chosen_vals = ut.take(rule_value_list, chosen_rule_idxs)
    for ((key, rule), val) in zip(chosen_rules, chosen_vals):
        if isinstance(rule, np.ndarray):
            flags = (rule == val)
        else:
            flags = rule(val)
        flags = np.logical_and(flags, participates)
        is_valid = np.logical_and(is_valid, flags)
        if verbose:
            verbinfo.print_post(is_valid, flags, ('SampleRule: %s = %r' % (key, val)))
    if allcfg:
        is_valid = np.logical_or(np.logical_or.reduce(is_valid.T)[:, None], is_valid)
        is_valid = np.logical_and(is_valid, participates)
    (qx_list, cfgx_list) = np.nonzero(is_valid)
    if (sortdsc is not None):
        assert (orderby is None), 'use orderby or sortasc'
        assert (reverse is None), 'reverse does not work with sortdsc'
        orderby = sortdsc
        reverse = True
    elif (sortasc is not None):
        assert (reverse is None), 'reverse does not work with sortasc'
        assert (orderby is None), 'use orderby or sortasc'
        orderby = sortasc
        reverse = False
    else:
        reverse = False
    if (orderby is not None):
        import re
        order_values = None
        for prefix_pattern in ['^gt_?', '^gf_?']:
            prefix_match = re.match(prefix_pattern, orderby)
            if (prefix_match is not None):
                truth = prefix_pattern[1:3]
                propname = orderby[prefix_match.end():]
                if verbose:
                    logger.info(('Ordering by truth=%s propname=%s' % (truth, propname)))
                order_values = truth2_prop[truth][propname]
                break
        if (order_values is None):
            raise NotImplementedError(('Unknown orerby=%r' % (orderby,)))
    else:
        order_values = np.arange(is_valid.size).reshape(is_valid.shape)
    flat_order = order_values[is_valid]
    if verbose:
        if verbose:
            logger.info('Reversing ordering (descending)')
        else:
            logger.info('Normal ordering (ascending)')
    if reverse:
        sortx = flat_order.argsort()[::(- 1)]
    else:
        sortx = flat_order.argsort()
    qx_list = qx_list.take(sortx, axis=0)
    cfgx_list = cfgx_list.take(sortx, axis=0)
    if (max_pername is not None):
        if verbose:
            logger.info(('Returning at most %d cases per name ' % (max_pername,)))
        _qaid_list = np.take(qaids, qx_list)
        _qnid_list = ibs.get_annot_nids(_qaid_list)
        _valid_idxs = []
        seen_ = ut.ddict((lambda : 0))
        for (idx, _qnid) in enumerate(_qnid_list):
            if (seen_[_qnid] < max_pername):
                seen_[_qnid] += 1
                _valid_idxs.append(idx)
        _qx_list = qx_list[_valid_idxs]
        _cfgx_list = cfgx_list[_valid_idxs]
        _valid_index = np.vstack((_qx_list, _cfgx_list)).T
        is_valid = vt.index_to_boolmask(_valid_index, is_valid.shape, isflat=False)
        qx_list = _qx_list
        cfgx_list = _cfgx_list
    if require_all_cfg:
        if verbose:
            prev_num_valid = is_valid.sum()
            logger.info('Enforcing that all configs must pass filters')
            logger.info(('  * prev_num_valid = %r' % (prev_num_valid,)))
        qx2_valid_cfgs = ut.group_items(cfgx_list, qx_list)
        hasall_cfg = [(len(qx2_valid_cfgs[qx]) == testres.nConfig) for qx in qx_list]
        _qx_list = qx_list.compress(hasall_cfg)
        _cfgx_list = cfgx_list.compress(hasall_cfg)
        _valid_index = np.vstack((_qx_list, _cfgx_list)).T
        is_valid = vt.index_to_boolmask(_valid_index, is_valid.shape, isflat=False)
        qx_list = _qx_list
        cfgx_list = _cfgx_list
        if verbose:
            verbinfo.print_post(is_valid, None, 'Enforcing that all configs must pass filters')
    if (index is not None):
        if isinstance(index, str):
            index = ut.smart_cast(index, slice)
        _qx_list = ut.take(qx_list, index)
        _cfgx_list = ut.take(cfgx_list, index)
        _valid_index = np.vstack((_qx_list, _cfgx_list)).T
        is_valid = vt.index_to_boolmask(_valid_index, is_valid.shape, isflat=False)
        qx_list = _qx_list
        cfgx_list = _cfgx_list
        if verbose:
            verbinfo.print_post(is_valid, None, ('Taking index=%r sample from len(qx_list) = %r' % (index, len(qx_list))))
    if (not return_mask):
        case_pos_list = np.vstack((qx_list, cfgx_list)).T
        case_identifier = case_pos_list
    else:
        if verbose:
            logger.info('Converting cases indicies to a 2d-mask')
        case_identifier = is_valid
    if verbose:
        logger.info('Finished case filtering')
        logger.info('Final case stats:')
        qx_hist = ut.dict_hist(qx_list)
        logger.info(('config per query stats: %r' % (ut.get_stats_str(qx_hist.values()),)))
        logger.info(('query per config stats: %r' % (ut.get_stats_str(ut.dict_hist(cfgx_list).values()),)))
    return case_identifier