class_docstr: Class which encapsulates logit of the master.
docstr: Computes results (scores, stats, etc...) of competition evaluation.
  
  Results are saved into output directory (self.results_dir).
  Also this method saves all intermediate data into output directory as well,
  so it can resume computation if it was interrupted for some reason.
  This is useful because computatin of resuls could take many minutes.
code: def compute_results(self):
      """Computes results (scores, stats, etc...) of competition evaluation.
  
      Results are saved into output directory (self.results_dir).
      Also this method saves all intermediate data into output directory as well,
      so it can resume computation if it was interrupted for some reason.
      This is useful because computatin of resuls could take many minutes.
      """
      # read all data
      logging.info("Reading data from datastore")
      dataset_meta = self._read_dataset_metadata()
      self.submissions.init_from_datastore()
      self.dataset_batches.init_from_datastore()
      self.adv_batches.init_from_datastore()
      self.attack_work.read_all_from_datastore()
  
      if os.path.exists(
          os.path.join(self.results_dir, "defense_work.dump")
      ):
          with open(
              os.path.join(self.results_dir, "defense_work.dump")
          ) as f:
              self.defense_work.deserialize(f)
      else:
          self.defense_work.read_all_from_datastore()
          with open(
              os.path.join(self.results_dir, "defense_work.dump"), "w"
          ) as f:
              self.defense_work.serialize(f)
  
      if os.path.exists(
          os.path.join(self.results_dir, "class_batches.dump")
      ):
          with open(
              os.path.join(self.results_dir, "class_batches.dump")
          ) as f:
              self.class_batches.deserialize(f)
      else:
          self.class_batches.init_from_datastore()
          with open(
              os.path.join(self.results_dir, "class_batches.dump"), "w"
          ) as f:
              self.class_batches.serialize(f)
  
      # process data
      logging.info("Processing classification results")
      count_adv_images = self.adv_batches.count_generated_adv_examples()
      intermediate_files = [
          "acc_matrix.dump",
          "error_matrix.dump",
          "hit_tc_matrix.dump",
          "classified_images_count.dump",
      ]
      if all(
          [
              os.path.exists(os.path.join(self.results_dir, fname))
              for fname in intermediate_files
          ]
      ):
          with open(
              os.path.join(self.results_dir, "acc_matrix.dump")
          ) as f:
              acc_matrix = pickle.load(f)
          with open(
              os.path.join(self.results_dir, "error_matrix.dump")
          ) as f:
              error_matrix = pickle.load(f)
          with open(
              os.path.join(self.results_dir, "hit_tc_matrix.dump")
          ) as f:
              hit_tc_matrix = pickle.load(f)
          with open(
              os.path.join(
                  self.results_dir, "classified_images_count.dump"
              )
          ) as f:
              classified_images_count = pickle.load(f)
      else:
          (
              acc_matrix,
              error_matrix,
              hit_tc_matrix,
              classified_images_count,
          ) = self.class_batches.compute_classification_results(
              self.adv_batches,
              self.dataset_batches,
              dataset_meta,
              self.defense_work,
          )
          with open(
              os.path.join(self.results_dir, "acc_matrix.dump"), "w"
          ) as f:
              pickle.dump(acc_matrix, f)
          with open(
              os.path.join(self.results_dir, "error_matrix.dump"), "w"
          ) as f:
              pickle.dump(error_matrix, f)
          with open(
              os.path.join(self.results_dir, "hit_tc_matrix.dump"), "w"
          ) as f:
              pickle.dump(hit_tc_matrix, f)
          with open(
              os.path.join(
                  self.results_dir, "classified_images_count.dump"
              ),
              "w",
          ) as f:
              pickle.dump(classified_images_count, f)
  
      # compute attacks and defenses which will be used for scoring
      logging.info(
          "Computing attacks and defenses which are used for scoring"
      )
      expected_num_adv_images = self.dataset_batches.count_num_images()
      attacks_to_use = [
          k
          for k, v in iteritems(count_adv_images)
          if (
              (v == expected_num_adv_images)
              and (k not in self.blacklisted_submissions)
          )
      ]
  
      total_num_adversarial = sum(itervalues(count_adv_images))
      defenses_to_use = [
          k
          for k, v in iteritems(classified_images_count)
          if (
              (v == total_num_adversarial)
              and (k not in self.blacklisted_submissions)
          )
      ]
  
      logging.info(
          "Expected number of adversarial images: %d",
          expected_num_adv_images,
      )
      logging.info(
          "Number of attacks to use to score defenses: %d",
          len(attacks_to_use),
      )
      logging.info(
          "Expected number of classification predictions: %d",
          total_num_adversarial,
      )
      logging.info(
          "Number of defenses to use to score attacks: %d",
          len(defenses_to_use),
      )
  
      save_dict_to_file(
          os.path.join(self.results_dir, "count_adv_images.csv"),
          count_adv_images,
      )
      save_dict_to_file(
          os.path.join(self.results_dir, "classified_images_count.csv"),
          classified_images_count,
      )
  
      # compute scores
      logging.info("Computing scores")
      attack_scores = defaultdict(lambda: 0)
      targeted_attack_scores = defaultdict(lambda: 0)
      defense_scores = defaultdict(lambda: 0)
      for defense_id in acc_matrix.dim0:
          for attack_id in acc_matrix.dim1:
              if attack_id in attacks_to_use:
                  defense_scores[defense_id] += acc_matrix[
                      defense_id, attack_id
                  ]
              if defense_id in defenses_to_use:
                  if attack_id in self.submissions.targeted_attacks:
                      targeted_attack_scores[
                          attack_id
                      ] += hit_tc_matrix[defense_id, attack_id]
                  else:
                      attack_scores[attack_id] += error_matrix[
                          defense_id, attack_id
                      ]
      # negate results of blacklisted submissions
      for s_id in self.blacklisted_submissions:
          if s_id in defense_scores:
              defense_scores[s_id] = -defense_scores[s_id]
          if s_id in attack_scores:
              attack_scores[s_id] = -attack_scores[s_id]
          if s_id in targeted_attack_scores:
              targeted_attack_scores[s_id] = -targeted_attack_scores[
                  s_id
              ]
      # save results
      logging.info("Saving results")
      all_attack_stats = self.attack_work.compute_work_statistics()
      nontargeted_attack_stats = {
          k: v
          for k, v in iteritems(all_attack_stats)
          if k in self.submissions.attacks
      }
      targeted_attack_stats = {
          k: v
          for k, v in iteritems(all_attack_stats)
          if k in self.submissions.targeted_attacks
      }
      defense_stats = self.defense_work.compute_work_statistics()
      self._save_work_results(
          nontargeted_attack_stats,
          attack_scores,
          count_adv_images,
          os.path.join(self.results_dir, "attack_results.csv"),
      )
      self._save_work_results(
          targeted_attack_stats,
          targeted_attack_scores,
          count_adv_images,
          os.path.join(self.results_dir, "targeted_attack_results.csv"),
      )
      self._save_work_results(
          defense_stats,
          defense_scores,
          classified_images_count,
          os.path.join(self.results_dir, "defense_results.csv"),
      )
  
      self._save_sorted_results(
          nontargeted_attack_stats,
          attack_scores,
          count_adv_images,
          os.path.join(self.results_dir, "sorted_attack_results.csv"),
      )
      self._save_sorted_results(
          targeted_attack_stats,
          targeted_attack_scores,
          count_adv_images,
          os.path.join(
              self.results_dir, "sorted_target_attack_results.csv"
          ),
      )
      self._save_sorted_results(
          defense_stats,
          defense_scores,
          classified_images_count,
          os.path.join(self.results_dir, "sorted_defense_results.csv"),
      )
  
      defense_id_to_name = {
          k: self.submissions.get_external_id(k)
          for k in iterkeys(self.submissions.defenses)
      }
      attack_id_to_name = {
          k: self.submissions.get_external_id(k)
          for k in self.submissions.get_all_attack_ids()
      }
      acc_matrix.save_to_file(
          os.path.join(self.results_dir, "accuracy_matrix.csv"),
          remap_dim0=defense_id_to_name,
          remap_dim1=attack_id_to_name,
      )
      error_matrix.save_to_file(
          os.path.join(self.results_dir, "error_matrix.csv"),
          remap_dim0=defense_id_to_name,
          remap_dim1=attack_id_to_name,
      )
      hit_tc_matrix.save_to_file(
          os.path.join(self.results_dir, "hit_target_class_matrix.csv"),
          remap_dim0=defense_id_to_name,
          remap_dim1=attack_id_to_name,
      )
  
      save_dict_to_file(
          os.path.join(self.results_dir, "defense_id_to_name.csv"),
          defense_id_to_name,
      )
      save_dict_to_file(
          os.path.join(self.results_dir, "attack_id_to_name.csv"),
          attack_id_to_name,
      )
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import argparse
  import collections
  from collections import defaultdict
  import csv
  from io import BytesIO
  import logging
  import os
  import pickle
  import random
  import time
  from six import iteritems
  from six import iterkeys
  from six import itervalues
  from six.moves import input
  import eval_lib
label_desc: Use image node_id 8 for calling the EvaluationMaster obj's underlying member method code with example usage: obj.compute_results() without return types