text: #   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  """
  Fluid Metrics
  
  The metrics are accomplished via Python natively. 
  """
  import numpy as np
  import copy
  import warnings
  
  __all__ = [
      'MetricBase',
      'CompositeMetric',
      'Accuracy',
      'ChunkEvaluator',
      'EditDistance',
      'DetectionMAP',
      'Auc',
  ]
  
  
  def _is_numpy_(var):
      return isinstance(var, (np.ndarray, np.generic))
  
  
  def _is_number_(var):
      return isinstance(var, int) or isinstance(var, float) or (isinstance(
          var, np.ndarray) and var.shape == (1, ))
  
  
  def _is_number_or_matrix_(var):
      return _is_number_(var) or isinstance(var, np.ndarray)
  
  
  class MetricBase(object):
      """
      Base Class for all evaluators
  
      Args:
          name(str): The name of evaluator. such as, "accuracy". Used for generate
              temporary variable name.
      Interface:
          Note(*) : the states is the attributes who not has _ prefix.
  
          get_config(): print current states and configuration
          reset(): clear the states. If the Metrics states type is not (int, float, np.ndarray),
                  Please override this method.
          update(): update states at every minibatch
          eval(): get metric evaluation in numpy type.
      """
  
      def __init__(self, name, **kwargs):
          self._name = str(name) if name != None else self.__class__.__name__
          self._kwargs = kwargs if kwargs != None else dict()
          self.reset()
  
      def __str__(self):
          return self._name
  
      def reset(self):
          """
          states is the attributes who not has _ prefix.
          reset the states of metrics.
          """
          states = {
              attr: value
              for attr, value in self.__dict__.iteritems()
              if not attr.startswith("_")
          }
          for attr, value in states.iteritems():
              if isinstance(value, int):
                  setattr(self, attr, 0)
              elif isinstance(value, float):
                  setattr(self, attr, .0)
              elif isinstance(value, (np.ndarray, np.generic)):
                  setattr(self, attr, np.zeros_like(value))
              else:
                  setattr(self, attr, None)
  
      def get_config(self):
          states = {
              attr: value
              for attr, value in self.__dict__.iteritems()
              if not attr.startswith("_")
          }
          config = copy.deepcopy(self._kwargs)
          config.update({"name": self._name, "states": copy.deepcopy(states)})
          return config
  
      def update(self):
          raise NotImplementedError()
  
      def eval(self):
          raise NotImplementedError()
  
  
  class CompositeMetric(MetricBase):
      """
      Compute multiple metrics in each minibatch.
      for example, merge F1, accuracy, recall into one Metric.
      """
  
      def __init__(self, name=None, **kwargs):
          super(CompositeMetric, self).__init__(name, kwargs)
          self._metrics = []
  
      def add_metric(self, metric):
          if not isinstance(metric, MetricBase):
              raise ValueError("SubMetric should be inherit from MetricBase.")
          self._metrics.append(metric)
  
      def eval(self):
          ans = []
          for m in self._metrics:
              ans.append(m.eval())
          return ans
  
  
  class Accuracy(MetricBase):
      """
      Accumulate the accuracy from minibatches and compute the average accuracy
      for every pass.
  
      Args:
         name: the metrics name
  
      Example:
          minibatch_accuracy = fluid.layers.accuracy(pred, label)
          accuracy_evaluator = fluid.metrics.Accuracy()
          for epoch in PASS_NUM:
              accuracy_evaluator.reset()
              for data in batches:
                  loss = exe.run(fetch_list=[cost, minibatch_accuracy])
              accuracy_evaluator.update(value=minibatch_accuracy, weight=batches)
              accuracy = accuracy_evaluator.eval()
      """
  
      def __init__(self, name=None):
          super(Accuracy, self).__init__(name)
          self.value = .0
          self.weight = .0
  
      def update(self, value, weight):
          if not _is_number_or_matrix_(value):
              raise ValueError(
                  "The 'value' must be a number(int, float) or a numpy ndarray.")
          if not _is_number_(weight):
              raise ValueError("The 'weight' must be a number(int, float).")
          self.value += value * weight
          self.weight += weight
  
      def eval(self):
          if self.weight == 0:
              raise ValueError(
                  "There is no data in Accuracy Metrics. Please check layers.accuracy output has added to Accuracy."
              )
          return self.value / self.weight
  
  
  class ChunkEvalutor(MetricBase):
      """
      Accumulate counter numbers output by chunk_eval from mini-batches and
      compute the precision recall and F1-score using the accumulated counter
      numbers.
      """
  
      def __init__(self, name=None):
          super(ChunkEvalutor, self).__init__(name)
          self.num_infer_chunks = 0
          self.num_label_chunks = 0
          self.num_correct_chunks = 0
  
      def update(self, num_infer_chunks, num_label_chunks, num_correct_chunks):
          if not _is_number_or_matrix_(num_infer_chunks):
              raise ValueError(
                  "The 'num_infer_chunks' must be a number(int, float) or a numpy ndarray."
              )
          if not _is_number_or_matrix_(num_label_chunks):
              raise ValueError(
                  "The 'num_label_chunks' must be a number(int, float) or a numpy ndarray."
              )
          if not _is_number_or_matrix_(num_correct_chunks):
              raise ValueError(
                  "The 'num_correct_chunks' must be a number(int, float) or a numpy ndarray."
              )
          self.num_infer_chunks += num_infer_chunks
          self.num_label_chunks += num_label_chunks
          self.num_correct_chunks += num_correct_chunks
  
      def eval(self):
          precision = float(
              self.num_correct_chunks
          ) / self.num_infer_chunks if self.num_infer_chunks else 0
          recall = float(self.num_correct_chunks
                         ) / self.num_label_chunks if self.num_label_chunks else 0
          f1_score = float(2 * precision * recall) / (
              precision + recall) if self.num_correct_chunks else 0
          return precision, recall, f1_score
  
  
  class EditDistance(MetricBase):
      """
      Accumulate edit distance sum and sequence number from mini-batches and
      compute the average edit_distance and instance error of all batches.
  
      Args:
          name: the metrics name
  
      Example:
          edit_distance_metrics = fluid.layers.edit_distance(input, label)
          distance_evaluator = fluid.metrics.EditDistance()
          for epoch in PASS_NUM:
              distance_evaluator.reset()
              for data in batches:
                  loss = exe.run(fetch_list=[cost] + list(edit_distance_metrics))
              distance_evaluator.update(*edit_distance_metrics)
              distance, instance_error = distance_evaluator.eval()
  
          In the above example:
          'distance' is the average of the edit distance in a pass.
          'instance_error' is the instance error rate in a pass.
  
      """
  
      def __init__(self, name):
          super(EditDistance, self).__init__(name)
          self.total_distance = .0
          self.seq_num = 0
          self.instance_error = 0
  
      def update(self, distances, seq_num):
          if not _is_numpy_(distances):
              raise ValueError("The 'distances' must be a numpy ndarray.")
          if not _is_number_(seq_num):
              raise ValueError("The 'seq_num' must be a number(int, float).")
          seq_right_count = np.sum(distances == 0)
          total_distance = np.sum(distances)
          self.seq_num += seq_num
          self.instance_error += seq_num - seq_right_count
          self.total_distance += total_distance
  
      def eval():
          if self.seq_num == 0:
              raise ValueError(
                  "There is no data in EditDistance Metric. Please check layers.edit_distance output has been added to EditDistance."
              )
          avg_distance = self.total_distance / self.seq_num
          avg_instance_error = self.instance_error / self.seq_num
          return avg_distance, avg_instance_error
  
  
  class DetectionMAP(MetricBase):
      """
      Calculate the detection mean average precision (mAP).
  
      TODO (Dang Qingqing): update the following doc.
      The general steps are as follows:
      1. calculate the true positive and false positive according to the input
          of detection and labels.
      2. calculate mAP value, support two versions: '11 point' and 'integral'.
  
      Please get more information from the following articles:
        https://sanchom.wordpress.com/tag/average-precision/
        https://arxiv.org/abs/1512.02325
      """
  
      def __init__(self, name=None):
          super(DetectionMAP, self).__init__(name)
          # the current map value
          self.value = .0
  
      def update(self, value, weight):
          if not _is_number_or_matrix_(value):
              raise ValueError(
                  "The 'value' must be a number(int, float) or a numpy ndarray.")
          if not _is_number_(weight):
              raise ValueError("The 'weight' must be a number(int, float).")
          self.value += value
          self.weight += weight
  
      def eval(self):
          if self.weight == 0:
              raise ValueError(
                  "There is no data in DetectionMAP Metrics. "
                  "Please check layers.detection_map output has added to DetectionMAP."
              )
          return self.value / self.weight
  
  
  class Auc(MetricBase):
      """
      Auc Metrics which adapts to binary classification.
      Need to note that auc metrics compute the value via Python natively.
      If you concern the speed, please use the fluid.layers.auc instead.
  
      The `auc` function creates four local variables, `true_positives`,
        `true_negatives`, `false_positives` and `false_negatives` that are used to
        compute the AUC. To discretize the AUC curve, a linearly spaced set of
        thresholds is used to compute pairs of recall and precision values. The area
        under the ROC-curve is therefore computed using the height of the recall
        values by the false positive rate, while the area under the PR-curve is the
        computed using the height of the precision values by the recall.
  
      Args:
          name: metric name
          curve: Specifies the name of the curve to be computed, 'ROC' [default] or
            'PR' for the Precision-Recall-curve.
          num_thresholds: The number of thresholds to use when discretizing the roc
              curve.
  
      "NOTE: only implement the ROC curve type via Python now."
      """
  
      def __init__(self, name, curve='ROC', num_thresholds=200):
          super(MetricBase, self).__init__(name, curve, num_thresholds)
          self._curve = curve
          self._num_thresholds = num_thresholds
          self._epsilon = 1e-6
          self.tp_list = np.ndarray((num_thresholds, ))
          self.fn_list = np.ndarray((num_thresholds, ))
          self.tn_list = np.ndarray((num_thresholds, ))
          self.fp_list = np.ndarray((num_thresholds, ))
  
      def update(self, labels, predictions, axis=1):
          if not _is_numpy_(labels):
              raise ValueError("The 'labels' must be a numpy ndarray.")
          if not _is_numpy_(predictions):
              raise ValueError("The 'predictions' must be a numpy ndarray.")
  
          kepsilon = 1e-7  # to account for floating point imprecisions
          thresholds = [(i + 1) * 1.0 / (num_thresholds - 1)
                        for i in range(num_thresholds - 2)]
          thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon]
  
          # caculate TP, FN, TN, FP count
          for idx_thresh, thresh in enumerate(thresholds):
              tp, fn, tn, fp = 0, 0, 0, 0
              for i, lbl in enumerate(labels):
                  if lbl:
                      if predictions[i, 0] >= thresh:
                          tp += 1
                      else:
                          fn += 1
                  else:
                      if predictions[i, 0] >= thresh:
                          fp += 1
                      else:
                          tn += 1
              tp_list[idx_thresh] += tp
              fn_list[idx_thresh] += fn
              tn_list[idx_thresh] += tn
              fp_list[idx_thresh] += fp
  
      def eval(self):
          epsilon = self._epsilon
          num_thresholds = self._num_thresholds
          tpr = (tp_list.astype("float32") + epsilon) / (
              tp_list + fn_list + epsilon)
          fpr = fp_list.astype("float32") / (fp_list + tn_list + epsilon)
          rec = (tp_list.astype("float32") + epsilon) / (
              tp_list + fp_list + epsilon)
  
          x = fpr[:num_thresholds - 1] - fpr[1:]
          y = (tpr[:num_thresholds - 1] + tpr[1:]) / 2.0
          auc_value = np.sum(x * y)
          return auc_value
  