BERT_rationale_benchmark/metrics.py
def score_hard_rationale_predictions(truth: List[Rationale], pred: List[Rationale]) -> Dict[str, Dict[str, float]]:
    """Computes instance (annotation)-level micro/macro averaged F1s"""
    scores = dict()
    truth = set(truth)
    pred = set(pred)
    micro_prec = len(truth & pred) / len(pred)
    micro_rec = len(truth & pred) / len(truth)
    micro_f1 = _f1(micro_prec, micro_rec)
    scores['instance_micro'] = {
                                'p': micro_prec,
                                'r': micro_rec,
                                'f1': micro_f1,
                               }

    ann_to_rat = _keyed_rationale_from_list(truth)
    pred_to_rat = _keyed_rationale_from_list(pred)
    instances_to_scores = dict()
    for k in set(ann_to_rat.keys()) | (pred_to_rat.keys()):
        if len(pred_to_rat.get(k, set())) > 0:
            instance_prec = len(ann_to_rat.get(k, set()) & pred_to_rat.get(k, set())) / len(pred_to_rat[k])
        else:
            instance_prec = 0
        if len(ann_to_rat.get(k, set())) > 0:
            instance_rec = len(ann_to_rat.get(k, set()) & pred_to_rat.get(k, set())) / len(ann_to_rat[k])
        else:
            instance_rec = 0
        instance_f1 = _f1(instance_prec, instance_rec)
        instances_to_scores[k] = {
                                    'p': instance_prec,
                                    'r': instance_rec,
                                    'f1': instance_f1,
                                 }
    # these are calculated as sklearn would
    macro_prec = sum(instance['p'] for instance in instances_to_scores.values()) / len(instances_to_scores)
    macro_rec = sum(instance['r'] for instance in instances_to_scores.values()) / len(instances_to_scores)
    macro_f1 = sum(instance['f1'] for instance in instances_to_scores.values()) / len(instances_to_scores)

    f1_scores = [instance['f1'] for instance in instances_to_scores.values()]
    print(macro_f1,  np.argsort(f1_scores)[::-1])



    scores['instance_macro'] = {
                                'p': macro_prec,
                                'r': macro_rec,
                                'f1': macro_f1,
                               }
    return scores
Computes instance (annotation)-level micro/macro averaged F1s
Computes instance (annotation)-level micro/macro averaged F1s
Computes instance ( annotation ) - level micro / macro averaged F1s
def score_hard_rationale_predictions(truth: List[Rationale], pred: List[Rationale]) -> Dict[str, Dict[str, float]]:
    scores = dict()
    truth = set(truth)
    pred = set(pred)
    micro_prec = len(truth & pred) / len(pred)
    micro_rec = len(truth & pred) / len(truth)
    micro_f1 = _f1(micro_prec, micro_rec)
    scores['instance_micro'] = {
                                'p': micro_prec,
                                'r': micro_rec,
                                'f1': micro_f1,
                               }
    ann_to_rat = _keyed_rationale_from_list(truth)
    pred_to_rat = _keyed_rationale_from_list(pred)
    instances_to_scores = dict()
    for k in set(ann_to_rat.keys()) | (pred_to_rat.keys()):
        if len(pred_to_rat.get(k, set())) > 0:
            instance_prec = len(ann_to_rat.get(k, set()) & pred_to_rat.get(k, set())) / len(pred_to_rat[k])
        else:
            instance_prec = 0
        if len(ann_to_rat.get(k, set())) > 0:
            instance_rec = len(ann_to_rat.get(k, set()) & pred_to_rat.get(k, set())) / len(ann_to_rat[k])
        else:
            instance_rec = 0
        instance_f1 = _f1(instance_prec, instance_rec)
        instances_to_scores[k] = {
                                    'p': instance_prec,
                                    'r': instance_rec,
                                    'f1': instance_f1,
                                 }
    macro_prec = sum(instance['p'] for instance in instances_to_scores.values()) / len(instances_to_scores)
    macro_rec = sum(instance['r'] for instance in instances_to_scores.values()) / len(instances_to_scores)
    macro_f1 = sum(instance['f1'] for instance in instances_to_scores.values()) / len(instances_to_scores)
    f1_scores = [instance['f1'] for instance in instances_to_scores.values()]
    print(macro_f1,  np.argsort(f1_scores)[::-1])
    scores['instance_macro'] = {
                                'p': macro_prec,
                                'r': macro_rec,
                                'f1': macro_f1,
                               }
    return scores
def score_hard_rationale_predictions ( truth : List [ Rationale ] , pred : List [ Rationale ] ) -> Dict [ str , Dict [ str , float ] ] : scores = dict ( ) truth = set ( truth ) pred = set ( pred ) micro_prec = len ( truth & pred ) / len ( pred ) micro_rec = len ( truth & pred ) / len ( truth ) micro_f1 = _f1 ( micro_prec , micro_rec ) scores [ 'instance_micro' ] = { 'p' : micro_prec , 'r' : micro_rec , 'f1' : micro_f1 , } ann_to_rat = _keyed_rationale_from_list ( truth ) pred_to_rat = _keyed_rationale_from_list ( pred ) instances_to_scores = dict ( ) for k in set ( ann_to_rat . keys ( ) ) | ( pred_to_rat . keys ( ) ) : if len ( pred_to_rat . get ( k , set ( ) ) ) > 0 : instance_prec = len ( ann_to_rat . get ( k , set ( ) ) & pred_to_rat . get ( k , set ( ) ) ) / len ( pred_to_rat [ k ] ) else : instance_prec = 0 if len ( ann_to_rat . get ( k , set ( ) ) ) > 0 : instance_rec = len ( ann_to_rat . get ( k , set ( ) ) & pred_to_rat . get ( k , set ( ) ) ) / len ( ann_to_rat [ k ] ) else : instance_rec = 0 instance_f1 = _f1 ( instance_prec , instance_rec ) instances_to_scores [ k ] = { 'p' : instance_prec , 'r' : instance_rec , 'f1' : instance_f1 , } macro_prec = sum ( instance [ 'p' ] for instance in instances_to_scores . values ( ) ) / len ( instances_to_scores ) macro_rec = sum ( instance [ 'r' ] for instance in instances_to_scores . values ( ) ) / len ( instances_to_scores ) macro_f1 = sum ( instance [ 'f1' ] for instance in instances_to_scores . values ( ) ) / len ( instances_to_scores ) f1_scores = [ instance [ 'f1' ] for instance in instances_to_scores . values ( ) ] print ( macro_f1 , np . argsort ( f1_scores ) [ : : - 1 ] ) scores [ 'instance_macro' ] = { 'p' : macro_prec , 'r' : macro_rec , 'f1' : macro_f1 , } return scores
Computes instance (annotation)-level micro/macro averaged F1s
Computes instance ( annotation ) - level micro / macro averaged F1s
"""Computes instance (annotation)-level micro/macro averaged F1s""" # these are calculated as sklearn would
truth
List[Rationale]
pred
List[Rationale]