text: # Copyright 2016-2020 The Van Valen Lab at the California Institute of
  # Technology (Caltech), with support from the Paul Allen Family Foundation,
  # Google, & National Institutes of Health (NIH) under Grant U24CA224309-01.
  # All rights reserved.
  #
  # Licensed under a modified Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.github.com/vanvalenlab/caliban-toolbox/LICENSE
  #
  # The Work provided may be used for non-commercial academic purposes only.
  # For any other use of the Work, including commercial use, please contact:
  # vanvalenlab@gmail.com
  #
  # Neither the name of Caltech nor the names of its contributors may be used
  # to endorse or promote products derived from this software without specific
  # prior written permission.
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  import numpy as np
  
  from deepcell_toolbox.metrics import Metrics, stats_pixelbased
  from scipy.stats import hmean
  
  
  class DatasetBenchmarker(object):
      """Class to perform benchmarking across different tissue and platform types
  
      Args:
          y_true: true labels
          y_pred: predicted labels
          tissue_list: list of tissue names for each image
          platform_list: list of platform names for each image
          model_name: name of the model used to generate the predictions
          metrics_kwargs: arguments to be passed to metrics package
  
      Raises:
          ValueError: if y_true and y_pred have different shapes
          ValueError: if y_true and y_pred are not 4D
          ValueError: if tissue_ids or platform_ids is not same length as labels
      """
      def __init__(self,
                   y_true,
                   y_pred,
                   tissue_list,
                   platform_list,
                   model_name,
                   metrics_kwargs={}):
          if y_true.shape != y_pred.shape:
              raise ValueError('Shape mismatch: y_true has shape {}, '
                               'y_pred has shape {}. Labels must have the same'
                               'shape.'.format(y_true.shape, y_pred.shape))
          if len(y_true.shape) != 4:
              raise ValueError('Data must be 4D, supplied data is {}'.format(y_true.shape))
  
          self.y_true = y_true
          self.y_pred = y_pred
  
          if len({y_true.shape[0], len(tissue_list), len(platform_list)}) != 1:
              raise ValueError('Tissue_list and platform_list must have same length as labels')
  
          self.tissue_list = tissue_list
          self.platform_list = platform_list
          self.model_name = model_name
          self.metrics = Metrics(model_name, **metrics_kwargs)
  
      def _benchmark_category(self, category_ids):
          """Compute benchmark stats over the different categories in supplied list
  
          Args:
              category_ids: list specifying which category each image belongs to
  
          Returns:
              dict: benchmarking results across each category
          """
  
          unique_categories = np.unique(category_ids)
  
          # create dict to hold stats across each category
          stats_dict = {}
          for cat in unique_categories:
              # cat the index of metrics corresponding to current category
              cat_idx = np.isin(category_ids, cat)
              cat_dict = {}
  
              # sum metrics across individual images within current category
              for key in self.metrics.stats:
                  cat_dict[key] = self.metrics.stats[key][cat_idx].sum()
  
              # compute additional metrics not produced by Metrics class
              cat_dict['recall'] = cat_dict['correct_detections'] / cat_dict['n_true']
  
              cat_dict['precision'] = cat_dict['correct_detections'] / cat_dict['n_pred']
  
              cat_dict['f1'] = hmean([cat_dict['recall'], cat_dict['precision']])
  
              pixel_stats = stats_pixelbased(self.y_true[cat_idx] != 0,
                                             self.y_pred[cat_idx] != 0)
              cat_dict['jaccard'] = pixel_stats['jaccard']
  
              # save current category dict to overall dict
              stats_dict[cat] = cat_dict
  
          return stats_dict
  
      def benchmark(self):
          self.metrics.calc_object_stats(self.y_true, self.y_pred)
  
          tissue_stats = self._benchmark_category(category_ids=self.tissue_list)
          platform_stats = self._benchmark_category(category_ids=self.platform_list)
          all_stats = self._benchmark_category(category_ids=['all'] * len(self.tissue_list))
          tissue_stats['all'] = all_stats['all']
          platform_stats['all'] = all_stats['all']
  
          return tissue_stats, platform_stats
  