class_docstr: Class to compile results from static turn annotations.
  
  Change PROBLEM_BUCKETS in task_config/annotations_config.json to be the buckets that
  you are asking crowdsource workers to annotate with.
  
  NOTE this script directly accesses Mephisto files rather than using the DataBrowser.
  This makes it fragile and not a great example for extension. It is a good candidate
  for refactor.
class_docstr_tok: ["Class","to","compile","results","from","static","turn","annotations",".","Change","PROBLEM_BUCKETS","in","task_config\/annotations_config.json","to","be","the","buckets","that","you","are","asking","crowdsource","workers","to","annotate","with",".","NOTE","this","script","directly","accesses","Mephisto","files","rather","than","using","the","DataBrowser",".","This","makes","it","fragile","and","not","a","great","example","for","extension",".","It","is","a","good","candidate","for","refactor","."]
docstr: Expects a df of index, rater_id, item_id and "data" column with each row a label
  of one of the categories.
docstr_tok: ["Expects","a","df","of","index",",","rater_id",",","item_id","and","``","data","''","column","with","each","row","a","label","of","one","of","the","categories","."]
code: def compute_fleiss_kappa(
      self, df: pd.DataFrame, categories: list, number_of_raters: int
  ) -> float:
      """
      Expects a df of index, rater_id, item_id and "data" column with each row a label
      of one of the categories.
      """
  
      # categories are "True" and "False"
      # As per wikipedia definition: https://en.wikipedia.org/wiki/Fleiss%27_kappa
      items_list = df.drop_duplicates(subset=["item_id"])[
          "item_id"
      ].to_list()
      N = len(items_list)
      p_j = np.zeros(len(categories))
      P_bar_sum_term = 0.0
      for item in items_list:
          df_item_annotations = df[df["item_id"] == item]
          if len(df_item_annotations) != number_of_raters:
              continue
          for j, c in enumerate(categories):
              try:
                  n_ij = df_item_annotations["data"].value_counts()[c]
              except Exception:
                  n_ij = 0.0
              p_j[j] += n_ij
              P_bar_sum_term += n_ij**2
  
      p_j = [tmp / (N * number_of_raters) for tmp in p_j]
      P_e_bar = sum([tmp**2 for tmp in p_j])
  
      P_bar = (P_bar_sum_term - N * number_of_raters) / (
          N * number_of_raters * (number_of_raters - 1)
      )
  
      kappa = (P_bar - P_e_bar) / (1 - P_e_bar)
      return kappa
  
imports: import copy
  import json
  import os
  from datetime import datetime
  from typing import Any, Dict, Optional, Tuple
  import numpy
  import pandas
  from parlai.crowdsourcing.utils.analysis import AbstractTurnAnnotationResultsCompiler
label_desc: Use image node_id 16 for calling the TurnAnnotationsStaticResultsCompiler obj's underlying member method code with example usage: obj.compute_fleiss_kappa(df, categories, number_of_raters) and returns: kappa