content: #!/usr/bin/env python
  # -*- coding: utf-8 -*-
  
  import sys
  import numpy
  import math
  from sklearn import metrics
  
  class DCASE2016_SceneClassification_Metrics():
      """DCASE 2016 scene classification metrics
  
      Examples
      --------
  
          >>> dcase2016_scene_metric = DCASE2016_SceneClassification_Metrics(class_list=dataset.scene_labels)
          >>> for fold in dataset.folds(mode=dataset_evaluation_mode):
          >>>     results = []
          >>>     result_filename = get_result_filename(fold=fold, path=result_path)
          >>>
          >>>     if os.path.isfile(result_filename):
          >>>         with open(result_filename, 'rt') as f:
          >>>             for row in csv.reader(f, delimiter='\t'):
          >>>                 results.append(row)
          >>>
          >>>     y_true = []
          >>>     y_pred = []
          >>>     for result in results:
          >>>         y_true.append(dataset.file_meta(result[0])[0]['scene_label'])
          >>>         y_pred.append(result[1])
          >>>
          >>>     dcase2016_scene_metric.evaluate(system_output=y_pred, annotated_ground_truth=y_true)
          >>>
          >>> results = dcase2016_scene_metric.results()
  
      """
  
      def __init__(self, class_list):
          """__init__ method.
  
          Parameters
          ----------
          class_list : list
              Evaluated scene labels in the list
  
          """
          self.accuracies_per_class = None
          self.Nsys = None
          self.Nref = None
          self.class_list = class_list
          self.eps = numpy.spacing(1)
  
      def __enter__(self):
          return self
  
      def __exit__(self, type, value, traceback):
          return self.results()
  
      def accuracies(self, y_true, y_pred, labels):
          """Calculate accuracy
  
          Parameters
          ----------
          y_true : numpy.array
              Ground truth array, list of scene labels
  
          y_pred : numpy.array
              System output array, list of scene labels
  
          labels : list
              list of scene labels
  
          Returns
          -------
          array : numpy.array [shape=(number of scene labels,)]
              Accuracy per scene label class
  
          """
  
          confusion_matrix = metrics.confusion_matrix(y_true=y_true, y_pred=y_pred, labels=labels).astype(float)
          #print confusion_matrix
          temp =  numpy.divide(numpy.diag(confusion_matrix), numpy.sum(confusion_matrix, 1)+self.eps)
          #print temp
          return temp
   
      def evaluate(self, annotated_ground_truth, system_output):
          """Evaluate system output and annotated ground truth pair.
  
          Use results method to get results.
  
          Parameters
          ----------
          annotated_ground_truth : numpy.array
              Ground truth array, list of scene labels
  
          system_output : numpy.array
              System output array, list of scene labels
  
          Returns
          -------
          nothing
  
          """
  
          accuracies_per_class = self.accuracies(y_pred=system_output, y_true=annotated_ground_truth, labels=self.class_list)
  
          if self.accuracies_per_class is None:
              self.accuracies_per_class = accuracies_per_class
          else:
              self.accuracies_per_class = numpy.vstack((self.accuracies_per_class, accuracies_per_class))
  
          Nref = numpy.zeros(len(self.class_list))
          Nsys = numpy.zeros(len(self.class_list))
  
          for class_id, class_label in enumerate(self.class_list):
              for item in system_output:
                  if item == class_label:
                      Nsys[class_id] += 1
  
              for item in annotated_ground_truth:
                  if item == class_label:
                      Nref[class_id] += 1
  
          if self.Nref is None:
              self.Nref = Nref
          else:
              self.Nref = numpy.vstack((self.Nref, Nref))
  
          if self.Nsys is None:
              self.Nsys = Nsys
          else:
              self.Nsys = numpy.vstack((self.Nsys, Nsys))
  
      def results(self):
          """Get results
  
          Outputs results in dict, format:
  
              {
                  'class_wise_data':
                      {
                          'office': {
                              'Nsys': 10,
                              'Nref': 7,
                          },
                      }
                  'class_wise_accuracy':
                      {
                          'office': 0.6,
                          'home': 0.4,
                      }
                  'overall_accuracy': numpy.mean(self.accuracies_per_class)
                  'Nsys': 100,
                  'Nref': 100,
              }
  
          Parameters
          ----------
          nothing
  
          Returns
          -------
          results : dict
              Results dict
  
          """
  
          results = {
              'class_wise_data': {},
              'class_wise_accuracy': {},
              'overall_accuracy': numpy.mean(self.accuracies_per_class)
          }
          if len(self.Nsys.shape) == 2:
              results['Nsys'] = int(sum(sum(self.Nsys)))
              results['Nref'] = int(sum(sum(self.Nref)))
          else:
              results['Nsys'] = int(sum(self.Nsys))
              results['Nref'] = int(sum(self.Nref))
  
          for class_id, class_label in enumerate(self.class_list):
              if len(self.accuracies_per_class.shape) == 2:
                  results['class_wise_accuracy'][class_label] = numpy.mean(self.accuracies_per_class[:, class_id])
                  results['class_wise_data'][class_label] = {
                     'Nsys': int(sum(self.Nsys[:, class_id])),
                      'Nref': int(sum(self.Nref[:, class_id])),
                  }
              else:
                  results['class_wise_accuracy'][class_label] = numpy.mean(self.accuracies_per_class[class_id])
                  results['class_wise_data'][class_label] = {
                     'Nsys': int(self.Nsys[class_id]),
                      'Nref': int(self.Nref[class_id]),
                  }
  
          return results
  
  
  class EventDetectionMetrics(object):
      """Baseclass for sound event metric classes.
      """
  
      def __init__(self, class_list):
          """__init__ method.
  
          Parameters
          ----------
          class_list : list
              List of class labels to be evaluated.
  
          """
  
          self.class_list = class_list
          self.eps = numpy.spacing(1)
  
      def max_event_offset(self, data):
          """Get maximum event offset from event list
  
          Parameters
          ----------
          data : list
              Event list, list of event dicts
  
          Returns
          -------
          max : float > 0
              Maximum event offset
          """
  
          max = 0
          for event in data:
              if event['event_offset'] > max:
                  max = event['event_offset']
          return max
  
      def list_to_roll(self, data, time_resolution=0.01):
          """Convert event list into event roll.
          Event roll is binary matrix indicating event activity withing time segment defined by time_resolution.
  
          Parameters
          ----------
          data : list
              Event list, list of event dicts
  
          time_resolution : float > 0
              Time resolution used when converting event into event roll.
  
          Returns
          -------
          event_roll : numpy.ndarray [shape=(math.ceil(data_length * 1 / time_resolution) + 1, amount of classes)]
              Event roll
          """
  
          # Initialize
          data_length = self.max_event_offset(data)
          event_roll = numpy.zeros((math.ceil(data_length * 1 / time_resolution) + 1, len(self.class_list)))
  
          # Fill-in event_roll
          for event in data:
              pos = self.class_list.index(event['event_label'].rstrip())
  
              onset = math.floor(event['event_onset'] * 1 / time_resolution)
              offset = math.ceil(event['event_offset'] * 1 / time_resolution) + 1
  
              event_roll[onset:offset, pos] = 1
  
          return event_roll
  
  
  class DCASE2016_EventDetection_SegmentBasedMetrics(EventDetectionMetrics):
      """DCASE2016 Segment based metrics for sound event detection
  
      Supported metrics:
      - Overall
          - Error rate (ER), Substitutions (S), Insertions (I), Deletions (D)
          - F-score (F1)
      - Class-wise
          - Error rate (ER), Insertions (I), Deletions (D)
          - F-score (F1)
  
      Examples
      --------
  
      >>> overall_metrics_per_scene = {}
      >>> for scene_id, scene_label in enumerate(dataset.scene_labels):
      >>>     dcase2016_segment_based_metric = DCASE2016_EventDetection_SegmentBasedMetrics(class_list=dataset.event_labels(scene_label=scene_label))
      >>>     for fold in dataset.folds(mode=dataset_evaluation_mode):
      >>>         results = []
      >>>         result_filename = get_result_filename(fold=fold, scene_label=scene_label, path=result_path)
      >>>
      >>>         if os.path.isfile(result_filename):
      >>>             with open(result_filename, 'rt') as f:
      >>>                 for row in csv.reader(f, delimiter='\t'):
      >>>                     results.append(row)
      >>>
      >>>         for file_id, item in enumerate(dataset.test(fold,scene_label=scene_label)):
      >>>             current_file_results = []
      >>>             for result_line in results:
      >>>                 if result_line[0] == dataset.absolute_to_relative(item['file']):
      >>>                     current_file_results.append(
      >>>                         {'file': result_line[0],
      >>>                          'event_onset': float(result_line[1]),
      >>>                          'event_offset': float(result_line[2]),
      >>>                          'event_label': result_line[3]
      >>>                          }
      >>>                     )
      >>>             meta = dataset.file_meta(dataset.absolute_to_relative(item['file']))
      >>>         dcase2016_segment_based_metric.evaluate(system_output=current_file_results, annotated_ground_truth=meta)
      >>> overall_metrics_per_scene[scene_label]['segment_based_metrics'] = dcase2016_segment_based_metric.results()
  
      """
  
      def __init__(self, class_list, time_resolution=1.0):
          """__init__ method.
  
          Parameters
          ----------
          class_list : list
              List of class labels to be evaluated.
  
          time_resolution : float > 0
              Time resolution used when converting event into event roll.
              (Default value = 1.0)
  
          """
  
          self.time_resolution = time_resolution
  
          self.overall = {
              'Ntp': 0.0,
              'Ntn': 0.0,
              'Nfp': 0.0,
              'Nfn': 0.0,
              'Nref': 0.0,
              'Nsys': 0.0,
              'ER': 0.0,
              'S': 0.0,
              'D': 0.0,
              'I': 0.0,
          }
          self.class_wise = {}
  
          for class_label in class_list:
              self.class_wise[class_label] = {
                  'Ntp': 0.0,
                  'Ntn': 0.0,
                  'Nfp': 0.0,
                  'Nfn': 0.0,
                  'Nref': 0.0,
                  'Nsys': 0.0,
              }
  
          EventDetectionMetrics.__init__(self, class_list=class_list)
  
      def __enter__(self):
          # Initialize class and return it
          return self
  
      def __exit__(self, type, value, traceback):
          # Finalize evaluation and return results
          return self.results()
  
      def evaluate(self, annotated_ground_truth, system_output):
          """Evaluate system output and annotated ground truth pair.
  
          Use results method to get results.
  
          Parameters
          ----------
          annotated_ground_truth : numpy.array
              Ground truth array, list of scene labels
  
          system_output : numpy.array
              System output array, list of scene labels
  
          Returns
          -------
          nothing
  
          """
  
          # Convert event list into frame-based representation
          system_event_roll = self.list_to_roll(data=system_output, time_resolution=self.time_resolution)
          annotated_event_roll = self.list_to_roll(data=annotated_ground_truth, time_resolution=self.time_resolution)
  
          # Fix durations of both event_rolls to be equal
          if annotated_event_roll.shape[0] > system_event_roll.shape[0]:
              padding = numpy.zeros((annotated_event_roll.shape[0] - system_event_roll.shape[0], len(self.class_list)))
              system_event_roll = numpy.vstack((system_event_roll, padding))
  
          if system_event_roll.shape[0] > annotated_event_roll.shape[0]:
              padding = numpy.zeros((system_event_roll.shape[0] - annotated_event_roll.shape[0], len(self.class_list)))
              annotated_event_roll = numpy.vstack((annotated_event_roll, padding))
  
          # Compute segment-based overall metrics
          for segment_id in range(0, annotated_event_roll.shape[0]):
              annotated_segment = annotated_event_roll[segment_id, :]
              system_segment = system_event_roll[segment_id, :]
  
              Ntp = sum(system_segment + annotated_segment > 1)
              Ntn = sum(system_segment + annotated_segment == 0)
              Nfp = sum(system_segment - annotated_segment > 0)
              Nfn = sum(annotated_segment - system_segment > 0)
  
              Nref = sum(annotated_segment)
              Nsys = sum(system_segment)
  
              S = min(Nref, Nsys) - Ntp
              D = max(0, Nref - Nsys)
              I = max(0, Nsys - Nref)
              ER = max(Nref, Nsys) - Ntp
  
              self.overall['Ntp'] += Ntp
              self.overall['Ntn'] += Ntn
              self.overall['Nfp'] += Nfp
              self.overall['Nfn'] += Nfn
              self.overall['Nref'] += Nref
              self.overall['Nsys'] += Nsys
              self.overall['S'] += S
              self.overall['D'] += D
              self.overall['I'] += I
              self.overall['ER'] += ER
  
          for class_id, class_label in enumerate(self.class_list):
              annotated_segment = annotated_event_roll[:, class_id]
              system_segment = system_event_roll[:, class_id]
  
              Ntp = sum(system_segment + annotated_segment > 1)
              Ntn = sum(system_segment + annotated_segment == 0)
              Nfp = sum(system_segment - annotated_segment > 0)
              Nfn = sum(annotated_segment - system_segment > 0)
  
              Nref = sum(annotated_segment)
              Nsys = sum(system_segment)
  
              self.class_wise[class_label]['Ntp'] += Ntp
              self.class_wise[class_label]['Ntn'] += Ntn
              self.class_wise[class_label]['Nfp'] += Nfp
              self.class_wise[class_label]['Nfn'] += Nfn
              self.class_wise[class_label]['Nref'] += Nref
              self.class_wise[class_label]['Nsys'] += Nsys
  
          return self
  
      def results(self):
          """Get results
  
          Outputs results in dict, format:
  
              {
                  'overall':
                      {
                          'Pre':
                          'Rec':
                          'F':
                          'ER':
                          'S':
                          'D':
                          'I':
                      }
                  'class_wise':
                      {
                          'office': {
                              'Pre':
                              'Rec':
                              'F':
                              'ER':
                              'D':
                              'I':
                              'Nref':
                              'Nsys':
                              'Ntp':
                              'Nfn':
                              'Nfp':
                          },
                      }
                  'class_wise_average':
                      {
                          'F':
                          'ER':
                      }
              }
  
          Parameters
          ----------
          nothing
  
          Returns
          -------
          results : dict
              Results dict
  
          """
  
          results = {'overall': {},
                     'class_wise': {},
                     'class_wise_average': {},
                     }
  
          # Overall metrics
          results['overall']['Pre'] = self.overall['Ntp'] / (self.overall['Nsys'] + self.eps)
          results['overall']['Rec'] = self.overall['Ntp'] / self.overall['Nref']
          results['overall']['F'] = 2 * ((results['overall']['Pre'] * results['overall']['Rec']) / (results['overall']['Pre'] + results['overall']['Rec'] + self.eps))
  
          results['overall']['ER'] = self.overall['ER'] / self.overall['Nref']
          results['overall']['S'] = self.overall['S'] / self.overall['Nref']
          results['overall']['D'] = self.overall['D'] / self.overall['Nref']
          results['overall']['I'] = self.overall['I'] / self.overall['Nref']
  
          # Class-wise metrics
          class_wise_F = []
          class_wise_ER = []
          for class_id, class_label in enumerate(self.class_list):
              if class_label not in results['class_wise']:
                  results['class_wise'][class_label] = {}
              results['class_wise'][class_label]['Pre'] = self.class_wise[class_label]['Ntp'] / (self.class_wise[class_label]['Nsys'] + self.eps)
              results['class_wise'][class_label]['Rec'] = self.class_wise[class_label]['Ntp'] / (self.class_wise[class_label]['Nref'] + self.eps)
              results['class_wise'][class_label]['F'] = 2 * ((results['class_wise'][class_label]['Pre'] * results['class_wise'][class_label]['Rec']) / (results['class_wise'][class_label]['Pre'] + results['class_wise'][class_label]['Rec'] + self.eps))
  
              results['class_wise'][class_label]['ER'] = (self.class_wise[class_label]['Nfn'] + self.class_wise[class_label]['Nfp']) / (self.class_wise[class_label]['Nref'] + self.eps)
              results['class_wise'][class_label]['D'] = self.class_wise[class_label]['Nfn'] / (self.class_wise[class_label]['Nref'] + self.eps)
              results['class_wise'][class_label]['I'] = self.class_wise[class_label]['Nfp'] / (self.class_wise[class_label]['Nref'] + self.eps)
  
              results['class_wise'][class_label]['Nref'] = self.class_wise[class_label]['Nref']
              results['class_wise'][class_label]['Nsys'] = self.class_wise[class_label]['Nsys']
              results['class_wise'][class_label]['Ntp'] = self.class_wise[class_label]['Ntp']
              results['class_wise'][class_label]['Nfn'] = self.class_wise[class_label]['Nfn']
              results['class_wise'][class_label]['Nfp'] = self.class_wise[class_label]['Nfp']
  
              class_wise_F.append(results['class_wise'][class_label]['F'])
              class_wise_ER.append(results['class_wise'][class_label]['ER'])
  
          results['class_wise_average']['F'] = numpy.mean(class_wise_F)
          results['class_wise_average']['ER'] = numpy.mean(class_wise_ER)
  
          return results
  
  
  class DCASE2016_EventDetection_EventBasedMetrics(EventDetectionMetrics):
      """DCASE2016 Event based metrics for sound event detection
  
      Supported metrics:
      - Overall
          - Error rate (ER), Substitutions (S), Insertions (I), Deletions (D)
          - F-score (F1)
      - Class-wise
          - Error rate (ER), Insertions (I), Deletions (D)
          - F-score (F1)
  
      Examples
      --------
  
      >>> overall_metrics_per_scene = {}
      >>> for scene_id, scene_label in enumerate(dataset.scene_labels):
      >>>     dcase2016_event_based_metric = DCASE2016_EventDetection_EventBasedMetrics(class_list=dataset.event_labels(scene_label=scene_label))
      >>>     for fold in dataset.folds(mode=dataset_evaluation_mode):
      >>>         results = []
      >>>         result_filename = get_result_filename(fold=fold, scene_label=scene_label, path=result_path)
      >>>
      >>>         if os.path.isfile(result_filename):
      >>>             with open(result_filename, 'rt') as f:
      >>>                 for row in csv.reader(f, delimiter='\t'):
      >>>                     results.append(row)
      >>>
      >>>         for file_id, item in enumerate(dataset.test(fold,scene_label=scene_label)):
      >>>             current_file_results = []
      >>>             for result_line in results:
      >>>                 if result_line[0] == dataset.absolute_to_relative(item['file']):
      >>>                     current_file_results.append(
      >>>                         {'file': result_line[0],
      >>>                          'event_onset': float(result_line[1]),
      >>>                          'event_offset': float(result_line[2]),
      >>>                          'event_label': result_line[3]
      >>>                          }
      >>>                     )
      >>>             meta = dataset.file_meta(dataset.absolute_to_relative(item['file']))
      >>>         dcase2016_event_based_metric.evaluate(system_output=current_file_results, annotated_ground_truth=meta)
      >>> overall_metrics_per_scene[scene_label]['event_based_metrics'] = dcase2016_event_based_metric.results()
  
      """
  
      def __init__(self, class_list, time_resolution=1.0, t_collar=0.2):
          """__init__ method.
  
          Parameters
          ----------
          class_list : list
              List of class labels to be evaluated.
  
          time_resolution : float > 0
              Time resolution used when converting event into event roll.
              (Default value = 1.0)
  
          t_collar : float > 0
              Time collar for event onset and offset condition
              (Default value = 0.2)
  
          """
  
          self.time_resolution = time_resolution
          self.t_collar = t_collar
  
          self.overall = {
              'Nref': 0.0,
              'Nsys': 0.0,
              'Nsubs': 0.0,
              'Ntp': 0.0,
              'Nfp': 0.0,
              'Nfn': 0.0,
          }
          self.class_wise = {}
  
          for class_label in class_list:
              self.class_wise[class_label] = {
                  'Nref': 0.0,
                  'Nsys': 0.0,
                  'Ntp': 0.0,
                  'Ntn': 0.0,
                  'Nfp': 0.0,
                  'Nfn': 0.0,
              }
  
          EventDetectionMetrics.__init__(self, class_list=class_list)
  
      def __enter__(self):
          # Initialize class and return it
          return self
  
      def __exit__(self, type, value, traceback):
          # Finalize evaluation and return results
          return self.results()
  
      def evaluate(self, annotated_ground_truth, system_output):
          """Evaluate system output and annotated ground truth pair.
  
          Use results method to get results.
  
          Parameters
          ----------
          annotated_ground_truth : numpy.array
              Ground truth array, list of scene labels
  
          system_output : numpy.array
              System output array, list of scene labels
  
          Returns
          -------
          nothing
  
          """
  
          # Overall metrics
  
          # Total number of detected and reference events
          Nsys = len(system_output)
          Nref = len(annotated_ground_truth)
  
          sys_correct = numpy.zeros(Nsys, dtype=bool)
          ref_correct = numpy.zeros(Nref, dtype=bool)
  
          # Number of correctly transcribed events, onset/offset within a t_collar range
          for j in range(0, len(annotated_ground_truth)):
              for i in range(0, len(system_output)):
                  label_condition = annotated_ground_truth[j]['event_label'] == system_output[i]['event_label']
                  onset_condition = self.onset_condition(annotated_event=annotated_ground_truth[j],
                                                         system_event=system_output[i],
                                                         t_collar=self.t_collar)
  
                  offset_condition = self.offset_condition(annotated_event=annotated_ground_truth[j],
                                                           system_event=system_output[i],
                                                           t_collar=self.t_collar)
  
                  if label_condition and onset_condition and offset_condition:
                      ref_correct[j] = True
                      sys_correct[i] = True
                      break
  
          Ntp = numpy.sum(sys_correct)
  
          sys_leftover = numpy.nonzero(numpy.negative(sys_correct))[0]
          ref_leftover = numpy.nonzero(numpy.negative(ref_correct))[0]
  
          # Substitutions
          Nsubs = 0
          for j in ref_leftover:
              for i in sys_leftover:
                  onset_condition = self.onset_condition(annotated_event=annotated_ground_truth[j],
                                                         system_event=system_output[i],
                                                         t_collar=self.t_collar)
  
                  offset_condition = self.offset_condition(annotated_event=annotated_ground_truth[j],
                                                           system_event=system_output[i],
                                                           t_collar=self.t_collar)
  
                  if onset_condition and offset_condition:
                      Nsubs += 1
                      break
  
          Nfp = Nsys - Ntp - Nsubs
          Nfn = Nref - Ntp - Nsubs
  
          self.overall['Nref'] += Nref
          self.overall['Nsys'] += Nsys
          self.overall['Ntp'] += Ntp
          self.overall['Nsubs'] += Nsubs
          self.overall['Nfp'] += Nfp
          self.overall['Nfn'] += Nfn
  
          # Class-wise metrics
          for class_id, class_label in enumerate(self.class_list):
              Nref = 0.0
              Nsys = 0.0
              Ntp = 0.0
  
              # Count event frequencies in the ground truth
              for i in range(0, len(annotated_ground_truth)):
                  if annotated_ground_truth[i]['event_label'] == class_label:
                      Nref += 1
  
              # Count event frequencies in the system output
              for i in range(0, len(system_output)):
                  if system_output[i]['event_label'] == class_label:
                      Nsys += 1
  
              for j in range(0, len(annotated_ground_truth)):
                  for i in range(0, len(system_output)):
                      if annotated_ground_truth[j]['event_label'] == class_label and system_output[i]['event_label'] == class_label:
                          onset_condition = self.onset_condition(annotated_event=annotated_ground_truth[j],
                                                                 system_event=system_output[i],
                                                                 t_collar=self.t_collar)
  
                          offset_condition = self.offset_condition(annotated_event=annotated_ground_truth[j],
                                                                   system_event=system_output[i],
                                                                   t_collar=self.t_collar)
  
                          if onset_condition and offset_condition:
                              Ntp += 1
                              break
  
              Nfp = Nsys - Ntp
              Nfn = Nref - Ntp
  
              self.class_wise[class_label]['Nref'] += Nref
              self.class_wise[class_label]['Nsys'] += Nsys
  
              self.class_wise[class_label]['Ntp'] += Ntp
              self.class_wise[class_label]['Nfp'] += Nfp
              self.class_wise[class_label]['Nfn'] += Nfn
  
      def onset_condition(self, annotated_event, system_event, t_collar=0.200):
          """Onset condition, checked does the event pair fulfill condition
  
          Condition:
  
          - event onsets are within t_collar each other
  
          Parameters
          ----------
          annotated_event : dict
              Event dict
  
          system_event : dict
              Event dict
  
          t_collar : float > 0
              Defines how close event onsets have to be in order to be considered match. In seconds.
              (Default value = 0.2)
  
          Returns
          -------
          result : bool
              Condition result
  
          """
  
          return math.fabs(annotated_event['event_onset'] - system_event['event_onset']) <= t_collar
  
      def offset_condition(self, annotated_event, system_event, t_collar=0.200, percentage_of_length=0.5):
          """Offset condition, checking does the event pair fulfill condition
  
          Condition:
  
          - event offsets are within t_collar each other
          or
          - system event offset is within the percentage_of_length*annotated event_length
  
          Parameters
          ----------
          annotated_event : dict
              Event dict
  
          system_event : dict
              Event dict
  
          t_collar : float > 0
              Defines how close event onsets have to be in order to be considered match. In seconds.
              (Default value = 0.2)
  
          percentage_of_length : float [0-1]
  
  
          Returns
          -------
          result : bool
              Condition result
  
          """
          annotated_length = annotated_event['event_offset'] - annotated_event['event_onset']
          return math.fabs(annotated_event['event_offset'] - system_event['event_offset']) <= max(t_collar, percentage_of_length * annotated_length)
  
      def results(self):
          """Get results
  
          Outputs results in dict, format:
  
              {
                  'overall':
                      {
                          'Pre':
                          'Rec':
                          'F':
                          'ER':
                          'S':
                          'D':
                          'I':
                      }
                  'class_wise':
                      {
                          'office': {
                              'Pre':
                              'Rec':
                              'F':
                              'ER':
                              'D':
                              'I':
                              'Nref':
                              'Nsys':
                              'Ntp':
                              'Nfn':
                              'Nfp':
                          },
                      }
                  'class_wise_average':
                      {
                          'F':
                          'ER':
                      }
              }
  
          Parameters
          ----------
          nothing
  
          Returns
          -------
          results : dict
              Results dict
  
          """
  
          results = {
              'overall': {},
              'class_wise': {},
              'class_wise_average': {},
          }
  
          # Overall metrics
          results['overall']['Pre'] = self.overall['Ntp'] / (self.overall['Nsys'] + self.eps)
          results['overall']['Rec'] = self.overall['Ntp'] / self.overall['Nref']
          results['overall']['F'] = 2 * ((results['overall']['Pre'] * results['overall']['Rec']) / (results['overall']['Pre'] + results['overall']['Rec'] + self.eps))
  
          results['overall']['ER'] = (self.overall['Nfn'] + self.overall['Nfp'] + self.overall['Nsubs']) / self.overall['Nref']
          results['overall']['S'] = self.overall['Nsubs'] / self.overall['Nref']
          results['overall']['D'] = self.overall['Nfn'] / self.overall['Nref']
          results['overall']['I'] = self.overall['Nfp'] / self.overall['Nref']
  
          # Class-wise metrics
          class_wise_F = []
          class_wise_ER = []
          
          for class_label in self.class_list:
              if class_label not in results['class_wise']:
                  results['class_wise'][class_label] = {}
  
              results['class_wise'][class_label]['Pre'] = self.class_wise[class_label]['Ntp'] / (self.class_wise[class_label]['Nsys'] + self.eps)
              results['class_wise'][class_label]['Rec'] = self.class_wise[class_label]['Ntp'] / (self.class_wise[class_label]['Nref'] + self.eps)
              results['class_wise'][class_label]['F'] = 2 * ((results['class_wise'][class_label]['Pre'] * results['class_wise'][class_label]['Rec']) / (results['class_wise'][class_label]['Pre'] + results['class_wise'][class_label]['Rec'] + self.eps))
  
              results['class_wise'][class_label]['ER'] = (self.class_wise[class_label]['Nfn']+self.class_wise[class_label]['Nfp']) / (self.class_wise[class_label]['Nref'] + self.eps)
              results['class_wise'][class_label]['D'] = self.class_wise[class_label]['Nfn'] / (self.class_wise[class_label]['Nref'] + self.eps)
              results['class_wise'][class_label]['I'] = self.class_wise[class_label]['Nfp'] / (self.class_wise[class_label]['Nref'] + self.eps)
  
              results['class_wise'][class_label]['Nref'] = self.class_wise[class_label]['Nref']
              results['class_wise'][class_label]['Nsys'] = self.class_wise[class_label]['Nsys']
              results['class_wise'][class_label]['Ntp'] = self.class_wise[class_label]['Ntp']
              results['class_wise'][class_label]['Nfn'] = self.class_wise[class_label]['Nfn']
              results['class_wise'][class_label]['Nfp'] = self.class_wise[class_label]['Nfp']
  
              class_wise_F.append(results['class_wise'][class_label]['F'])
              class_wise_ER.append(results['class_wise'][class_label]['ER'])
  
          # Class-wise average
          results['class_wise_average']['F'] = numpy.mean(class_wise_F)
          results['class_wise_average']['ER'] = numpy.mean(class_wise_ER)
  
          return results
  
  
  class DCASE2013_EventDetection_Metrics(EventDetectionMetrics):
      """Lecagy DCASE2013 metrics, converted from the provided Matlab implementation
  
      Supported metrics:
      - Frame based
          - F-score (F)
          - AEER
      - Event based
          - Onset
              - F-Score (F)
              - AEER
          - Onset-offset
              - F-Score (F)
              - AEER
      - Class based
          - Onset
              - F-Score (F)
              - AEER
          - Onset-offset
              - F-Score (F)
              - AEER
      """
  
      #
  
      def frame_based(self, annotated_ground_truth, system_output, resolution=0.01):
          # Convert event list into frame-based representation
          system_event_roll = self.list_to_roll(data=system_output, time_resolution=resolution)
          annotated_event_roll = self.list_to_roll(data=annotated_ground_truth, time_resolution=resolution)
  
          # Fix durations of both event_rolls to be equal
          if annotated_event_roll.shape[0] > system_event_roll.shape[0]:
              padding = numpy.zeros((annotated_event_roll.shape[0] - system_event_roll.shape[0], len(self.class_list)))
              system_event_roll = numpy.vstack((system_event_roll, padding))
  
          if system_event_roll.shape[0] > annotated_event_roll.shape[0]:
              padding = numpy.zeros((system_event_roll.shape[0] - annotated_event_roll.shape[0], len(self.class_list)))
              annotated_event_roll = numpy.vstack((annotated_event_roll, padding))
  
          # Compute frame-based metrics
          Nref = sum(sum(annotated_event_roll))
          Ntot = sum(sum(system_event_roll))
          Ntp = sum(sum(system_event_roll + annotated_event_roll > 1))
          Nfp = sum(sum(system_event_roll - annotated_event_roll > 0))
          Nfn = sum(sum(annotated_event_roll - system_event_roll > 0))
          Nsubs = min(Nfp, Nfn)
  
          eps = numpy.spacing(1)
  
          results = dict()
          results['Rec'] = Ntp / (Nref + eps)
          results['Pre'] = Ntp / (Ntot + eps)
          results['F'] = 2 * ((results['Pre'] * results['Rec']) / (results['Pre'] + results['Rec'] + eps))
          results['AEER'] = (Nfn + Nfp + Nsubs) / (Nref + eps)
  
          return results
  
      def event_based(self, annotated_ground_truth, system_output):
          # Event-based evaluation for event detection task
          # outputFile: the output of the event detection system
          # GTFile: the ground truth list of events
  
          # Total number of detected and reference events
          Ntot = len(system_output)
          Nref = len(annotated_ground_truth)
  
          # Number of correctly transcribed events, onset within a +/-100 ms range
          Ncorr = 0
          NcorrOff = 0
          for j in range(0, len(annotated_ground_truth)):
              for i in range(0, len(system_output)):
                  if annotated_ground_truth[j]['event_label'] == system_output[i]['event_label'] and (math.fabs(annotated_ground_truth[j]['event_onset'] - system_output[i]['event_onset']) <= 0.1):
                      Ncorr += 1
  
                      # If offset within a +/-100 ms range or within 50% of ground-truth event's duration
                      if math.fabs(annotated_ground_truth[j]['event_offset'] - system_output[i]['event_offset']) <= max(0.1, 0.5 * (annotated_ground_truth[j]['event_offset'] - annotated_ground_truth[j]['event_onset'])):
                          NcorrOff += 1
  
                      break  # In order to not evaluate duplicates
  
          # Compute onset-only event-based metrics
          eps = numpy.spacing(1)
          results = {
              'onset': {},
              'onset-offset': {},
          }
  
          Nfp = Ntot - Ncorr
          Nfn = Nref - Ncorr
          Nsubs = min(Nfp, Nfn)
          results['onset']['Rec'] = Ncorr / (Nref + eps)
          results['onset']['Pre'] = Ncorr / (Ntot + eps)
          results['onset']['F'] = 2 * (
              (results['onset']['Pre'] * results['onset']['Rec']) / (
                  results['onset']['Pre'] + results['onset']['Rec'] + eps))
          results['onset']['AEER'] = (Nfn + Nfp + Nsubs) / (Nref + eps)
  
          # Compute onset-offset event-based metrics
          NfpOff = Ntot - NcorrOff
          NfnOff = Nref - NcorrOff
          NsubsOff = min(NfpOff, NfnOff)
          results['onset-offset']['Rec'] = NcorrOff / (Nref + eps)
          results['onset-offset']['Pre'] = NcorrOff / (Ntot + eps)
          results['onset-offset']['F'] = 2 * ((results['onset-offset']['Pre'] * results['onset-offset']['Rec']) / (
              results['onset-offset']['Pre'] + results['onset-offset']['Rec'] + eps))
          results['onset-offset']['AEER'] = (NfnOff + NfpOff + NsubsOff) / (Nref + eps)
  
          return results
  
      def class_based(self, annotated_ground_truth, system_output):
          # Class-wise event-based evaluation for event detection task
          # outputFile: the output of the event detection system
          # GTFile: the ground truth list of events
  
          # Total number of detected and reference events per class
          Ntot = numpy.zeros((len(self.class_list), 1))
          for event in system_output:
              pos = self.class_list.index(event['event_label'])
              Ntot[pos] += 1
  
          Nref = numpy.zeros((len(self.class_list), 1))
          for event in annotated_ground_truth:
              pos = self.class_list.index(event['event_label'])
              Nref[pos] += 1
  
          I = (Nref > 0).nonzero()[0]  # index for classes present in ground-truth
  
          # Number of correctly transcribed events per class, onset within a +/-100 ms range
          Ncorr = numpy.zeros((len(self.class_list), 1))
          NcorrOff = numpy.zeros((len(self.class_list), 1))
  
          for j in range(0, len(annotated_ground_truth)):
              for i in range(0, len(system_output)):
                  if annotated_ground_truth[j]['event_label'] == system_output[i]['event_label'] and (
                              math.fabs(
                                      annotated_ground_truth[j]['event_onset'] - system_output[i]['event_onset']) <= 0.1):
                      pos = self.class_list.index(system_output[i]['event_label'])
                      Ncorr[pos] += 1
  
                      # If offset within a +/-100 ms range or within 50% of ground-truth event's duration
                      if math.fabs(annotated_ground_truth[j]['event_offset'] - system_output[i]['event_offset']) <= max(
                              0.1, 0.5 * (
                                          annotated_ground_truth[j]['event_offset'] - annotated_ground_truth[j][
                                          'event_onset'])):
                          pos = self.class_list.index(system_output[i]['event_label'])
                          NcorrOff[pos] += 1
  
                      break  # In order to not evaluate duplicates
  
          # Compute onset-only class-wise event-based metrics
          eps = numpy.spacing(1)
          results = {
              'onset': {},
              'onset-offset': {},
          }
  
          Nfp = Ntot - Ncorr
          Nfn = Nref - Ncorr
          Nsubs = numpy.minimum(Nfp, Nfn)
          tempRec = Ncorr[I] / (Nref[I] + eps)
          tempPre = Ncorr[I] / (Ntot[I] + eps)
          results['onset']['Rec'] = numpy.mean(tempRec)
          results['onset']['Pre'] = numpy.mean(tempPre)
          tempF = 2 * ((tempPre * tempRec) / (tempPre + tempRec + eps))
          results['onset']['F'] = numpy.mean(tempF)
          tempAEER = (Nfn[I] + Nfp[I] + Nsubs[I]) / (Nref[I] + eps)
          results['onset']['AEER'] = numpy.mean(tempAEER)
  
          # Compute onset-offset class-wise event-based metrics
          NfpOff = Ntot - NcorrOff
          NfnOff = Nref - NcorrOff
          NsubsOff = numpy.minimum(NfpOff, NfnOff)
          tempRecOff = NcorrOff[I] / (Nref[I] + eps)
          tempPreOff = NcorrOff[I] / (Ntot[I] + eps)
          results['onset-offset']['Rec'] = numpy.mean(tempRecOff)
          results['onset-offset']['Pre'] = numpy.mean(tempPreOff)
          tempFOff = 2 * ((tempPreOff * tempRecOff) / (tempPreOff + tempRecOff + eps))
          results['onset-offset']['F'] = numpy.mean(tempFOff)
          tempAEEROff = (NfnOff[I] + NfpOff[I] + NsubsOff[I]) / (Nref[I] + eps)
          results['onset-offset']['AEER'] = numpy.mean(tempAEEROff)
  
          return results
  
  
  def main(argv):
      # Examples to show usage and required data structures
      class_list = ['class1', 'class2', 'class3']
      system_output = [
          {
              'event_label': 'class1',
              'event_onset': 0.1,
              'event_offset': 1.0
          },
          {
              'event_label': 'class2',
              'event_onset': 4.1,
              'event_offset': 4.7
          },
          {
              'event_label': 'class3',
              'event_onset': 5.5,
              'event_offset': 6.7
          }
      ]
      annotated_groundtruth = [
          {
              'event_label': 'class1',
              'event_onset': 0.1,
              'event_offset': 1.0
          },
          {
              'event_label': 'class2',
              'event_onset': 4.2,
              'event_offset': 5.4
          },
          {
              'event_label': 'class3',
              'event_onset': 5.5,
              'event_offset': 6.7
          }
      ]
      dcase2013metric = DCASE2013_EventDetection_Metrics(class_list=class_list)
  
      print 'DCASE2013'
      print 'Frame-based:', dcase2013metric.frame_based(system_output=system_output,
                                                        annotated_ground_truth=annotated_groundtruth)
      print 'Event-based:', dcase2013metric.event_based(system_output=system_output,
                                                        annotated_ground_truth=annotated_groundtruth)
      print 'Class-based:', dcase2013metric.class_based(system_output=system_output,
                                                        annotated_ground_truth=annotated_groundtruth)
  
      dcase2016_metric = DCASE2016_EventDetection_SegmentBasedMetrics(class_list=class_list)
      print 'DCASE2016'
      print dcase2016_metric.evaluate(system_output=system_output, annotated_ground_truth=annotated_groundtruth).results()
  
  
  if __name__ == "__main__":
      sys.exit(main(sys.argv))
  