class_docstr: Class to compile results from static turn annotations.
  
  Change PROBLEM_BUCKETS in task_config/annotations_config.json to be the buckets that
  you are asking crowdsource workers to annotate with.
  
  NOTE this script directly accesses Mephisto files rather than using the DataBrowser.
  This makes it fragile and not a great example for extension. It is a good candidate
  for refactor.
class_docstr_tok: ["Class","to","compile","results","from","static","turn","annotations",".","Change","PROBLEM_BUCKETS","in","task_config\/annotations_config.json","to","be","the","buckets","that","you","are","asking","crowdsource","workers","to","annotate","with",".","NOTE","this","script","directly","accesses","Mephisto","files","rather","than","using","the","DataBrowser",".","This","makes","it","fragile","and","not","a","great","example","for","extension",".","It","is","a","good","candidate","for","refactor","."]
code: def calculate_basic_interannotator_agreement(
      self, df: pd.DataFrame
  ) -> None:
      print("Starting calculate_interannotator_agreement...")
      # Drops the human utterances which don't have problem buckets
      bot_only_df = df.replace("", np.nan)
      # Get rid of the None's, which occur if there were no checkboxes (so if
      # last utterance only option selected)
      bot_only_df = bot_only_df.fillna(value=np.nan)
      bot_only_df = bot_only_df.dropna()
  
      bot_only_df = self._problem_bucket_specific_filtering(bot_only_df)
  
      # Group at the utterance level (summing across workers)
      bot_only_df = bot_only_df.replace(True, 1)
      bot_only_df = bot_only_df.replace(False, 0)
      all_bot_annotations_count = len(bot_only_df)
  
      # Remove utterances that don't have self.NUM_ANNOTATIONS annotations
      counted_df = bot_only_df.groupby(["utterance_id"]).count()
      counted_df = counted_df[
          counted_df == self.NUM_ANNOTATIONS
      ].dropna()
      bot_only_df = bot_only_df[
          bot_only_df["utterance_id"].isin(counted_df.index)
      ]
      print(
          f"Removed {all_bot_annotations_count - len(bot_only_df)} that did not have annotations by {self.NUM_ANNOTATIONS} workers. {len(bot_only_df)} annotations remaining."
      )
  
      if self.LIVE_ONBOARDING_IS_LAST_SUBTASK:
          # Remove those that didn't get enough right on live onboarding
          bot_only_df = bot_only_df[
              bot_only_df["qc_success_pct"]
              >= self.LIVE_ONBOARDING_THRESHOLD
          ]
  
      summed_df = bot_only_df.groupby(["utterance_id"]).sum()
      print(
          f"summed_df has length {len(summed_df)}; bot_only_df: {len(bot_only_df)}"
      )
  
      utterance_ids = bot_only_df["utterance_id"].unique()
      print(f"Number of unique utterance_ids: {len(utterance_ids)}.")
  
      if "any_problem" in summed_df:
          # We've computed a column marking if any problem exists, so include this
          extended_problem_buckets = self.problem_buckets + [
              "any_problem"
          ]
      else:
          extended_problem_buckets = self.problem_buckets
      for k in extended_problem_buckets:
          one_annotator = len(summed_df[summed_df[k] == 1])
          two_annotators = len(summed_df[summed_df[k] == 2])
          three_annotators = len(summed_df[summed_df[k] >= 3])
          total_problem_annotations = (
              one_annotator + two_annotators + three_annotators
          )
          total_utterances = len(summed_df[k])
          if total_problem_annotations > 0:
              print(
                  f"Bucket: {k}, total unique problem utterances: {total_problem_annotations} ({total_problem_annotations/total_utterances:.1%} of all), one annotator: {one_annotator} ({one_annotator/total_problem_annotations:.1%}), two_annotators: {two_annotators} ({two_annotators/total_problem_annotations:.1%}), three+ annotators: {three_annotators} ({three_annotators/total_problem_annotations:.1%})"
              )
  
imports: import copy
  import json
  import os
  from datetime import datetime
  from typing import Any, Dict, Optional, Tuple
  import numpy
  import pandas
  from parlai.crowdsourcing.utils.analysis import AbstractTurnAnnotationResultsCompiler
label_desc: Use image node_id 12 for calling the TurnAnnotationsStaticResultsCompiler obj's underlying member method code with example usage: obj.calculate_basic_interannotator_agreement(df) without return types