scripts/significance_test.py
def evaluate(scores_m1, scores_m2, labels, CLASSES):
    """
    Evaluates the predicted classes w.r.t. a gold file.
    """

    vocab_map = dict([(i, v) for i, v in enumerate(CLASSES)])

    mlb = MultiLabelBinarizer()
    mlb.fit([CLASSES])
    # Hack to maintain order
    mlb.classes_ = np.array(CLASSES)

    gold_label = mlb.transform(labels.tolist())
    pred_score_m1 = np.matrix(scores_m1.tolist())

    pred_label_m1 = (pred_score_m1 == pred_score_m1.max(1)).astype(int)


    pred_score_m2 = np.matrix(scores_m2.tolist())
    pred_label_m2 = (pred_score_m2  == pred_score_m2.max(1)).astype(int)


    for i, clas in enumerate(classes):
        print(clas)
        res = mcnemar_test(pred_label_m1[:,i], pred_label_m2[:,i], np.transpose(np.matrix(gold_label[:,i])))
        print('statistic=%.3f, p-value=%.3f\n' % (res.statistic, res.pvalue))

    bowker_test(pred_label_m1, pred_label_m2, np.matrix(gold_label), CLASSES)
    return mcnemar_test(pred_label_m1, pred_label_m2, np.matrix(gold_label))

    Evaluates the predicted classes w.r.t. a gold file.
    
Evaluates the predicted classes w.r.t. a gold file.
Evaluates the predicted classes w . r . t . a gold file .
def evaluate(scores_m1, scores_m2, labels, CLASSES):
    vocab_map = dict([(i, v) for i, v in enumerate(CLASSES)])
    mlb = MultiLabelBinarizer()
    mlb.fit([CLASSES])
    mlb.classes_ = np.array(CLASSES)
    gold_label = mlb.transform(labels.tolist())
    pred_score_m1 = np.matrix(scores_m1.tolist())
    pred_label_m1 = (pred_score_m1 == pred_score_m1.max(1)).astype(int)
    pred_score_m2 = np.matrix(scores_m2.tolist())
    pred_label_m2 = (pred_score_m2  == pred_score_m2.max(1)).astype(int)
    for i, clas in enumerate(classes):
        print(clas)
        res = mcnemar_test(pred_label_m1[:,i], pred_label_m2[:,i], np.transpose(np.matrix(gold_label[:,i])))
        print('statistic=%.3f, p-value=%.3f\n' % (res.statistic, res.pvalue))
    bowker_test(pred_label_m1, pred_label_m2, np.matrix(gold_label), CLASSES)
    return mcnemar_test(pred_label_m1, pred_label_m2, np.matrix(gold_label))
def evaluate ( scores_m1 , scores_m2 , labels , CLASSES ) : vocab_map = dict ( [ ( i , v ) for i , v in enumerate ( CLASSES ) ] ) mlb = MultiLabelBinarizer ( ) mlb . fit ( [ CLASSES ] ) mlb . classes_ = np . array ( CLASSES ) gold_label = mlb . transform ( labels . tolist ( ) ) pred_score_m1 = np . matrix ( scores_m1 . tolist ( ) ) pred_label_m1 = ( pred_score_m1 == pred_score_m1 . max ( 1 ) ) . astype ( int ) pred_score_m2 = np . matrix ( scores_m2 . tolist ( ) ) pred_label_m2 = ( pred_score_m2 == pred_score_m2 . max ( 1 ) ) . astype ( int ) for i , clas in enumerate ( classes ) : print ( clas ) res = mcnemar_test ( pred_label_m1 [ : , i ] , pred_label_m2 [ : , i ] , np . transpose ( np . matrix ( gold_label [ : , i ] ) ) ) print ( 'statistic=%.3f, p-value=%.3f\n' % ( res . statistic , res . pvalue ) ) bowker_test ( pred_label_m1 , pred_label_m2 , np . matrix ( gold_label ) , CLASSES ) return mcnemar_test ( pred_label_m1 , pred_label_m2 , np . matrix ( gold_label ) )
Evaluates the predicted classes w.r.t.
Evaluates the predicted classes w . r . t .
"""
    Evaluates the predicted classes w.r.t. a gold file.
    """ # Hack to maintain order
scores_m1
scores_m2
labels
CLASSES