Spaces:
Sleeping
Sleeping
File size: 2,365 Bytes
519b419 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
# -*- coding:utf-8 -*-
import numpy as np
def fleiss_kappa_function(M):
"""Computes Fleiss' kappa for group of annotators.
:param M: a matrix of shape (:attr:'N', :attr:'k') with 'N' = number of subjects and 'k' = the number of categories.
'M[i, j]' represent the number of raters who assigned the 'i'th subject to the 'j'th category.
:type: numpy matrix
:rtype: float
:return: Fleiss' kappa score
"""
N, k = M.shape # N is # of items, k is # of categories
n_annotators = float(np.sum(M[0, :])) # # of annotators
tot_annotations = N * n_annotators # the total # of annotations
category_sum = np.sum(M, axis=0) # the sum of each category over all items
# chance agreement
p = category_sum / tot_annotations # the distribution of each category over all annotations
PbarE = np.sum(p * p) # average chance agreement over all categories
# observed agreement
P = (np.sum(M * M, axis=1) - n_annotators) / (n_annotators * (n_annotators - 1))
Pbar = np.sum(P) / N # add all observed agreement chances per item and divide by amount of items
return round((Pbar - PbarE) / (1 - PbarE), 4)
def cohen_kappa_function(ann1, ann2):
"""Computes Cohen kappa for pair-wise annotators.
:param ann1: annotations provided by first annotator
:type ann1: list
:param ann2: annotations provided by second annotator
:type ann2: list
:rtype: float
:return: Cohen kappa statistic
"""
count = 0
for an1, an2 in zip(ann1, ann2):
if an1 == an2:
count += 1
A = count / len(ann1) # observed agreement A (Po)
uniq = set(ann1 + ann2)
E = 0 # expected agreement E (Pe)
for item in uniq:
cnt1 = ann1.count(item)
cnt2 = ann2.count(item)
count = (cnt1 / len(ann1)) * (cnt2 / len(ann2))
E += count
return round((A - E) / (1 - E), 4)
def interpret_kappa(score):
color = ""
if score < 0:
color= "#e74c3c;"
elif 0.01 <= score <= 0.20:
color= "#f39c12;"
elif 0.21 <= score <= 0.40:
color= "#f4d03f;"
elif 0.41 <= score <= 0.60:
color= "#5dade2;"
elif 0.61 <= score <= 0.80:
color= "#58d68d;"
elif 0.81 <= score <= 0.99:
color= "#28b463;"
return f"<span style='font-size:30px; color: {color}'>{round(score*100, 2)} %</span>" |