class_docstr: Class to compile results from static turn annotations.
  
  Change PROBLEM_BUCKETS in task_config/annotations_config.json to be the buckets that
  you are asking crowdsource workers to annotate with.
  
  NOTE this script directly accesses Mephisto files rather than using the DataBrowser.
  This makes it fragile and not a great example for extension. It is a good candidate
  for refactor.
class_docstr_tok: ["Class","to","compile","results","from","static","turn","annotations",".","Change","PROBLEM_BUCKETS","in","task_config\/annotations_config.json","to","be","the","buckets","that","you","are","asking","crowdsource","workers","to","annotate","with",".","NOTE","this","script","directly","accesses","Mephisto","files","rather","than","using","the","DataBrowser",".","This","makes","it","fragile","and","not","a","great","example","for","extension",".","It","is","a","good","candidate","for","refactor","."]
code: def calculate_stats_interannotator_agreement(self, df: pd.DataFrame):
      print("Starting calculate_stats_interannotator_agreement...")
      # Get rid of the human utterances (non-annotated)
      bot_only_df = df.replace("", np.nan)
      # Get rid of the None's, which occur if there were no checkboxes (so if
      # last utterance only option selected)
      bot_only_df = bot_only_df.fillna(value=np.nan)
      bot_only_df = bot_only_df.dropna()
      print(f"Calculating agreement on {len(bot_only_df)} annotations.")
  
      for pb in self.problem_buckets:
          # Expects a df of rater_id, item_id and "data" column
          kappa_df = df[
              ["annotation_id", "worker_id", "utterance_id", pb]
          ]
          kappa_df = kappa_df.rename(
              columns={
                  "worker_id": "rater_id",
                  "utterance_id": "item_id",
                  pb: "data",
              }
          )
          try:
              fleiss_kappa = self.compute_fleiss_kappa(
                  kappa_df, [True, False], self.NUM_ANNOTATIONS
              )
          except Exception as exc:
              print(
                  f"Exception calculating Fleiss Kappa: {exc}. Skipping."
              )
              continue
          print(f"Fleiss' kappa for {pb} is: {fleiss_kappa:0.3f}.")
  
imports: import copy
  import json
  import os
  from datetime import datetime
  from typing import Any, Dict, Optional, Tuple
  import numpy
  import pandas
  from parlai.crowdsourcing.utils.analysis import AbstractTurnAnnotationResultsCompiler
label_desc: Use image node_id 15 for calling the TurnAnnotationsStaticResultsCompiler obj's underlying member method code with example usage: obj.calculate_stats_interannotator_agreement(df) without return types