class_docstr: Class to compile results from static turn annotations.
  
  Change PROBLEM_BUCKETS in task_config/annotations_config.json to be the buckets that
  you are asking crowdsource workers to annotate with.
  
  NOTE this script directly accesses Mephisto files rather than using the DataBrowser.
  This makes it fragile and not a great example for extension. It is a good candidate
  for refactor.
class_docstr_tok: ["Class","to","compile","results","from","static","turn","annotations",".","Change","PROBLEM_BUCKETS","in","task_config\/annotations_config.json","to","be","the","buckets","that","you","are","asking","crowdsource","workers","to","annotate","with",".","NOTE","this","script","directly","accesses","Mephisto","files","rather","than","using","the","DataBrowser",".","This","makes","it","fragile","and","not","a","great","example","for","extension",".","It","is","a","good","candidate","for","refactor","."]
docstr: Assume gold_annotations are a dictionary of the form {utterance_id : {bucket_i:
  true/false}} where utterance_id is taken from the compile_initial_results (i.e.
  mephistohitid_subtaskindex_utteranceidx)
docstr_tok: ["Assume","gold_annotations","are","a","dictionary","of","the","form","{","utterance_id",":","{","bucket_i",":","true\/false","}","}","where","utterance_id","is","taken","from","the","compile_initial_results","(","i.e",".","mephistohitid_subtaskindex_utteranceidx",")"]
code: def calculate_agreement_with_gold_annotations(
      self, gold_annotations, df: pd.DataFrame
  ) -> None:
      """
      Assume gold_annotations are a dictionary of the form {utterance_id : {bucket_i:
      true/false}} where utterance_id is taken from the compile_initial_results (i.e.
      mephistohitid_subtaskindex_utteranceidx)
      """
      print("Starting calculate_agreement_with_gold_annotations...")
      # Drops the human utterances which don't have problem buckets
      bot_only_df = df.replace("", np.nan)
      # Get rid of the None's, which occur if there were no checkboxes (so if
      # last utterance only option selected)
      bot_only_df = bot_only_df.fillna(value=np.nan)
      bot_only_df = bot_only_df.dropna()
  
      # Include only utterances that have gold_annotations
      bot_only_df = bot_only_df[
          bot_only_df["utterance_id"].isin(gold_annotations.keys())
      ]
  
      print(
          f"Got {len(gold_annotations.keys())} utterances with gold annotations. Found {len(bot_only_df)} utterances matching gold annotations from DataFrame."
      )
  
      agreement_map = {pb: [] for pb in self.problem_buckets}
      agreement_map_problem_only = {
          pb: [] for pb in self.problem_buckets
      }
      problem_counts = {pb: 0 for pb in self.problem_buckets}
      for utterance_id, gold in gold_annotations.items():
          utterance_df = bot_only_df[
              bot_only_df["utterance_id"] == utterance_id
          ]
          count_workers = len(utterance_df)
  
          for pb in self.problem_buckets:
              gold_annotation = gold[pb]
              match_count = utterance_df[
                  utterance_df[pb] == gold[pb]
              ].count()[pb]
              a = float(match_count / count_workers)
              agreement_map[pb].append(a)
              if gold_annotation:
                  agreement_map_problem_only[pb].append(a)
                  problem_counts[pb] += 1
      print(
          f"------------------------\nAverage agreement with {len(gold_annotations)} total gold utterances annotated was:"
      )
      for pb in self.problem_buckets:
          print(
              f"{pb}: {np.average(agreement_map[pb]):.1%} ({problem_counts[pb]} gold problem samples)"
          )
      print("------------------------")
      print(
          f"------------------------\nAverage agreement problem samples only with {len(gold_annotations)} total gold utterances annotated was:"
      )
      for pb in self.problem_buckets:
          print(
              f"{pb}: {np.average(agreement_map_problem_only[pb]):.1%} ({problem_counts[pb]} gold problem samples)"
          )
      print("------------------------")
  
imports: import copy
  import json
  import os
  from datetime import datetime
  from typing import Any, Dict, Optional, Tuple
  import numpy
  import pandas
  from parlai.crowdsourcing.utils.analysis import AbstractTurnAnnotationResultsCompiler
label_desc: Use image node_id 14 for calling the TurnAnnotationsStaticResultsCompiler obj's underlying member method code with example usage: obj.calculate_agreement_with_gold_annotations(gold_annotations, df) without return types