code: # Copyright 2016 Google Inc. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #      http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS-IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  """Provides functions to help with evaluating models."""
  import datetime
  import numpy
  
  from . import mean_average_precision_calculator as map_calculator
  from . import average_precision_calculator as ap_calculator
  
  
  def flatten(l):
      """ Merges a list of lists into a single list. """
      return [item for sublist in l for item in sublist]
  
  
  def calculate_hit_at_one(predictions, actuals):
      """Performs a local (numpy) calculation of the hit at one.
  
    Args:
      predictions: Matrix containing the outputs of the model.
        Dimensions are 'batch' x 'num_classes'.
      actuals: Matrix containing the ground truth labels.
        Dimensions are 'batch' x 'num_classes'.
  
    Returns:
      float: The average hit at one across the entire batch.
    """
      top_prediction = numpy.argmax(predictions, 1)
      hits = actuals[numpy.arange(actuals.shape[0]), top_prediction]
      return numpy.average(hits)
  
  
  def calculate_precision_at_equal_recall_rate(predictions, actuals):
      """Performs a local (numpy) calculation of the PERR.
  
    Args:
      predictions: Matrix containing the outputs of the model.
        Dimensions are 'batch' x 'num_classes'.
      actuals: Matrix containing the ground truth labels.
        Dimensions are 'batch' x 'num_classes'.
  
    Returns:
      float: The average precision at equal recall rate across the entire batch.
    """
      aggregated_precision = 0.0
      num_videos = actuals.shape[0]
      for row in numpy.arange(num_videos):
          num_labels = int(numpy.sum(actuals[row]))
          top_indices = numpy.argpartition(predictions[row],
                                           -num_labels)[-num_labels:]
          item_precision = 0.0
          for label_index in top_indices:
              if predictions[row][label_index] > 0:
                  item_precision += actuals[row][label_index]
          item_precision /= top_indices.size
          aggregated_precision += item_precision
      aggregated_precision /= num_videos
      return aggregated_precision
  
  
  def calculate_gap(predictions, actuals, top_k=20):
      """Performs a local (numpy) calculation of the global average precision.
  
    Only the top_k predictions are taken for each of the videos.
  
    Args:
      predictions: Matrix containing the outputs of the model.
        Dimensions are 'batch' x 'num_classes'.
      actuals: Matrix containing the ground truth labels.
        Dimensions are 'batch' x 'num_classes'.
      top_k: How many predictions to use per video.
  
    Returns:
      float: The global average precision.
    """
      gap_calculator = ap_calculator.AveragePrecisionCalculator()
      sparse_predictions, sparse_labels, num_positives = top_k_by_class(
          predictions, actuals, top_k)
      gap_calculator.accumulate(
          flatten(sparse_predictions), flatten(sparse_labels), sum(num_positives))
      return gap_calculator.peek_ap_at_n()
  
  
  def top_k_by_class(predictions, labels, k=20):
      """Extracts the top k predictions for each video, sorted by class.
  
    Args:
      predictions: A numpy matrix containing the outputs of the model.
        Dimensions are 'batch' x 'num_classes'.
      k: the top k non-zero entries to preserve in each prediction.
  
    Returns:
      A tuple (predictions,labels, true_positives). 'predictions' and 'labels'
      are lists of lists of floats. 'true_positives' is a list of scalars. The
      length of the lists are equal to the number of classes. The entries in the
      predictions variable are probability predictions, and
      the corresponding entries in the labels variable are the ground truth for
      those predictions. The entries in 'true_positives' are the number of true
      positives for each class in the ground truth.
  
    Raises:
      ValueError: An error occurred when the k is not a positive integer.
    """
      if k <= 0:
          raise ValueError("k must be a positive integer.")
      k = min(k, predictions.shape[1])
      num_classes = predictions.shape[1]
      prediction_triplets = []
      for video_index in range(predictions.shape[0]):
          prediction_triplets.extend(
              top_k_triplets(predictions[video_index], labels[video_index], k))
      out_predictions = [[] for v in range(num_classes)]
      out_labels = [[] for v in range(num_classes)]
      for triplet in prediction_triplets:
          out_predictions[triplet[0]].append(triplet[1])
          out_labels[triplet[0]].append(triplet[2])
      out_true_positives = [numpy.sum(labels[:, i]) for i in range(num_classes)]
  
      return out_predictions, out_labels, out_true_positives
  
  
  def top_k_triplets(predictions, labels, k=20):
      """Get the top_k for a 1-d numpy array. Returns a sparse list of tuples in
    (prediction, class) format"""
      m = len(predictions)
      k = min(k, m)
      indices = numpy.argpartition(predictions, -k)[-k:]
      return [(index, predictions[index], labels[index]) for index in indices]
  
  
  class EvaluationMetrics(object):
      """A class to store the evaluation metrics."""
  
      def __init__(self, num_class, top_k):
          """Construct an EvaluationMetrics object to store the evaluation metrics.
  
      Args:
        num_class: A positive integer specifying the number of classes.
        top_k: A positive integer specifying how many predictions are considered per video.
  
      Raises:
        ValueError: An error occurred when MeanAveragePrecisionCalculator cannot
          not be constructed.
      """
          self.sum_hit_at_one = 0.0
          self.sum_perr = 0.0
          self.sum_loss = 0.0
          self.map_calculator = map_calculator.MeanAveragePrecisionCalculator(
              num_class)
          self.global_ap_calculator = ap_calculator.AveragePrecisionCalculator()
          self.top_k = top_k
          self.num_examples = 0
  
      #def accumulate(self, predictions, labels, loss):
      def accumulate(self, loss, predictions, labels):
          """Accumulate the metrics calculated locally for this mini-batch.
  
      Args:
        predictions: A numpy matrix containing the outputs of the model.
          Dimensions are 'batch' x 'num_classes'.
        labels: A numpy matrix containing the ground truth labels.
          Dimensions are 'batch' x 'num_classes'.
        loss: A numpy array containing the loss for each sample.
  
      Returns:
        dictionary: A dictionary storing the metrics for the mini-batch.
  
      Raises:
        ValueError: An error occurred when the shape of predictions and actuals
          does not match.
      """
          batch_size = labels.shape[0]
          mean_hit_at_one = calculate_hit_at_one(predictions, labels)
          mean_perr = calculate_precision_at_equal_recall_rate(predictions,
                                                               labels)
          mean_loss = numpy.mean(loss)
  
          # Take the top 20 predictions.
          sparse_predictions, sparse_labels, num_positives = top_k_by_class(
              predictions, labels, self.top_k)
          self.map_calculator.accumulate(sparse_predictions, sparse_labels,
                                         num_positives)
          self.global_ap_calculator.accumulate(
              flatten(sparse_predictions),
              flatten(sparse_labels), sum(num_positives))
  
          self.num_examples += batch_size
          self.sum_hit_at_one += mean_hit_at_one * batch_size
          self.sum_perr += mean_perr * batch_size
          self.sum_loss += mean_loss * batch_size
  
          return {
              "hit_at_one": mean_hit_at_one,
              "perr": mean_perr,
              "loss": mean_loss
          }
  
      def get(self):
          """Calculate the evaluation metrics for the whole epoch.
  
      Raises:
        ValueError: If no examples were accumulated.
  
      Returns:
        dictionary: a dictionary storing the evaluation metrics for the epoch. The
          dictionary has the fields: avg_hit_at_one, avg_perr, avg_loss, and
          aps (default nan).
      """
          if self.num_examples <= 0:
              raise ValueError("total_sample must be positive.")
          avg_hit_at_one = self.sum_hit_at_one / self.num_examples
          avg_perr = self.sum_perr / self.num_examples
          avg_loss = self.sum_loss / self.num_examples
  
          aps = self.map_calculator.peek_map_at_n()
          gap = self.global_ap_calculator.peek_ap_at_n()
  
          epoch_info_dict = {}
          return {
              "avg_hit_at_one": avg_hit_at_one,
              "avg_perr": avg_perr,
              "avg_loss": avg_loss,
              "aps": aps,
              "gap": gap
          }
  
      def clear(self):
          """Clear the evaluation metrics and reset the EvaluationMetrics object."""
          self.sum_hit_at_one = 0.0
          self.sum_perr = 0.0
          self.sum_loss = 0.0
          self.map_calculator.clear()
          self.global_ap_calculator.clear()
          self.num_examples = 0
   #  Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
  #
  #Licensed under the Apache License, Version 2.0 (the "License");
  #you may not use this file except in compliance with the License.
  #You may obtain a copy of the License at
  #
  #    http://www.apache.org/licenses/LICENSE-2.0
  #
  #Unless required by applicable law or agreed to in writing, software
  #distributed under the License is distributed on an "AS IS" BASIS,
  #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #See the License for the specific language governing permissions and
  #limitations under the License.
  
  import paddle
  import paddle.fluid as fluid
  from paddle.fluid import ParamAttr
  import numpy as np
  import math
  
  from bmn_utils import get_interp1d_mask
  
  DATATYPE = 'float32'
  
  
  # Net
  class Conv1D(fluid.dygraph.Layer):
      def __init__(self,
                   prefix,
                   num_channels=256,
                   num_filters=256,
                   size_k=3,
                   padding=1,
                   groups=1,
                   act="relu"):
          super(Conv1D, self).__init__()
          fan_in = num_channels * size_k * 1
          k = 1. / math.sqrt(fan_in)
          param_attr = ParamAttr(
              name=prefix + "_w",
              initializer=fluid.initializer.Uniform(
                  low=-k, high=k))
          bias_attr = ParamAttr(
              name=prefix + "_b",
              initializer=fluid.initializer.Uniform(
                  low=-k, high=k))
  
          self._conv2d = fluid.dygraph.Conv2D(
              num_channels=num_channels,
              num_filters=num_filters,
              filter_size=(1, size_k),
              stride=1,
              padding=(0, padding),
              groups=groups,
              act=act,
              param_attr=param_attr,
              bias_attr=bias_attr)
  
      def forward(self, x):
          x = fluid.layers.unsqueeze(input=x, axes=[2])
          x = self._conv2d(x)
          x = fluid.layers.squeeze(input=x, axes=[2])
          return x
  
  
  class BMN(fluid.dygraph.Layer):
      def __init__(self, cfg):
          super(BMN, self).__init__()
  
          #init config
          self.tscale = cfg.MODEL.tscale
          self.dscale = cfg.MODEL.dscale
          self.prop_boundary_ratio = cfg.MODEL.prop_boundary_ratio
          self.num_sample = cfg.MODEL.num_sample
          self.num_sample_perbin = cfg.MODEL.num_sample_perbin
  
          self.hidden_dim_1d = 256
          self.hidden_dim_2d = 128
          self.hidden_dim_3d = 512
  
          # Base Module
          self.b_conv1 = Conv1D(
              prefix="Base_1",
              num_channels=400,
              num_filters=self.hidden_dim_1d,
              size_k=3,
              padding=1,
              groups=4,
              act="relu")
          self.b_conv2 = Conv1D(
              prefix="Base_2",
              num_filters=self.hidden_dim_1d,
              size_k=3,
              padding=1,
              groups=4,
              act="relu")
  
          # Temporal Evaluation Module
          self.ts_conv1 = Conv1D(
              prefix="TEM_s1",
              num_filters=self.hidden_dim_1d,
              size_k=3,
              padding=1,
              groups=4,
              act="relu")
          self.ts_conv2 = Conv1D(
              prefix="TEM_s2", num_filters=1, size_k=1, padding=0, act="sigmoid")
          self.te_conv1 = Conv1D(
              prefix="TEM_e1",
              num_filters=self.hidden_dim_1d,
              size_k=3,
              padding=1,
              groups=4,
              act="relu")
          self.te_conv2 = Conv1D(
              prefix="TEM_e2", num_filters=1, size_k=1, padding=0, act="sigmoid")
  
          #Proposal Evaluation Module
          self.p_conv1 = Conv1D(
              prefix="PEM_1d",
              num_filters=self.hidden_dim_2d,
              size_k=3,
              padding=1,
              act="relu")
  
          # init to speed up
          sample_mask = get_interp1d_mask(self.tscale, self.dscale,
                                          self.prop_boundary_ratio,
                                          self.num_sample, self.num_sample_perbin)
          self.sample_mask = fluid.dygraph.base.to_variable(sample_mask)
          self.sample_mask.stop_gradient = True
  
          self.p_conv3d1 = fluid.dygraph.Conv3D(
              num_channels=128,
              num_filters=self.hidden_dim_3d,
              filter_size=(self.num_sample, 1, 1),
              stride=(self.num_sample, 1, 1),
              padding=0,
              act="relu",
              param_attr=ParamAttr(name="PEM_3d1_w"),
              bias_attr=ParamAttr(name="PEM_3d1_b"))
  
          self.p_conv2d1 = fluid.dygraph.Conv2D(
              num_channels=512,
              num_filters=self.hidden_dim_2d,
              filter_size=1,
              stride=1,
              padding=0,
              act="relu",
              param_attr=ParamAttr(name="PEM_2d1_w"),
              bias_attr=ParamAttr(name="PEM_2d1_b"))
          self.p_conv2d2 = fluid.dygraph.Conv2D(
              num_channels=128,
              num_filters=self.hidden_dim_2d,
              filter_size=3,
              stride=1,
              padding=1,
              act="relu",
              param_attr=ParamAttr(name="PEM_2d2_w"),
              bias_attr=ParamAttr(name="PEM_2d2_b"))
          self.p_conv2d3 = fluid.dygraph.Conv2D(
              num_channels=128,
              num_filters=self.hidden_dim_2d,
              filter_size=3,
              stride=1,
              padding=1,
              act="relu",
              param_attr=ParamAttr(name="PEM_2d3_w"),
              bias_attr=ParamAttr(name="PEM_2d3_b"))
          self.p_conv2d4 = fluid.dygraph.Conv2D(
              num_channels=128,
              num_filters=2,
              filter_size=1,
              stride=1,
              padding=0,
              act="sigmoid",
              param_attr=ParamAttr(name="PEM_2d4_w"),
              bias_attr=ParamAttr(name="PEM_2d4_b"))
  
      def forward(self, x):
          #Base Module
          x = self.b_conv1(x)
          x = self.b_conv2(x)
  
          #TEM
          xs = self.ts_conv1(x)
          xs = self.ts_conv2(xs)
          xs = fluid.layers.squeeze(xs, axes=[1])
          xe = self.te_conv1(x)
          xe = self.te_conv2(xe)
          xe = fluid.layers.squeeze(xe, axes=[1])
  
          #PEM
          xp = self.p_conv1(x)
          #BM layer
          xp = fluid.layers.matmul(xp, self.sample_mask)
          xp = fluid.layers.reshape(
              xp, shape=[0, 0, -1, self.dscale, self.tscale])
  
          xp = self.p_conv3d1(xp)
          xp = fluid.layers.squeeze(xp, axes=[2])
          xp = self.p_conv2d1(xp)
          xp = self.p_conv2d2(xp)
          xp = self.p_conv2d3(xp)
          xp = self.p_conv2d4(xp)
          return xp, xs, xe
  
  
  def bmn_loss_func(pred_bm, pred_start, pred_end, gt_iou_map, gt_start, gt_end,
                    cfg):
      def _get_mask(cfg):
          dscale = cfg.MODEL.dscale
          tscale = cfg.MODEL.tscale
          bm_mask = []
          for idx in range(dscale):
              mask_vector = [1 for i in range(tscale - idx)
                             ] + [0 for i in range(idx)]
              bm_mask.append(mask_vector)
          bm_mask = np.array(bm_mask, dtype=np.float32)
          self_bm_mask = fluid.layers.create_global_var(
              shape=[dscale, tscale], value=0, dtype=DATATYPE, persistable=True)
          fluid.layers.assign(bm_mask, self_bm_mask)
          self_bm_mask.stop_gradient = True
          return self_bm_mask
  
      def tem_loss_func(pred_start, pred_end, gt_start, gt_end):
          def bi_loss(pred_score, gt_label):
              pred_score = fluid.layers.reshape(
                  x=pred_score, shape=[-1], inplace=False)
              gt_label = fluid.layers.reshape(
                  x=gt_label, shape=[-1], inplace=False)
              gt_label.stop_gradient = True
              pmask = fluid.layers.cast(x=(gt_label > 0.5), dtype=DATATYPE)
              num_entries = fluid.layers.cast(
                  fluid.layers.shape(pmask), dtype=DATATYPE)
              num_positive = fluid.layers.cast(
                  fluid.layers.reduce_sum(pmask), dtype=DATATYPE)
              ratio = num_entries / num_positive
              coef_0 = 0.5 * ratio / (ratio - 1)
              coef_1 = 0.5 * ratio
              epsilon = 0.000001
              temp = fluid.layers.log(pred_score + epsilon)
              loss_pos = fluid.layers.elementwise_mul(
                  fluid.layers.log(pred_score + epsilon), pmask)
              loss_pos = coef_1 * fluid.layers.reduce_mean(loss_pos)
              loss_neg = fluid.layers.elementwise_mul(
                  fluid.layers.log(1.0 - pred_score + epsilon), (1.0 - pmask))
              loss_neg = coef_0 * fluid.layers.reduce_mean(loss_neg)
              loss = -1 * (loss_pos + loss_neg)
              return loss
  
          loss_start = bi_loss(pred_start, gt_start)
          loss_end = bi_loss(pred_end, gt_end)
          loss = loss_start + loss_end
          return loss
  
      def pem_reg_loss_func(pred_score, gt_iou_map, mask):
  
          gt_iou_map = fluid.layers.elementwise_mul(gt_iou_map, mask)
  
          u_hmask = fluid.layers.cast(x=gt_iou_map > 0.7, dtype=DATATYPE)
          u_mmask = fluid.layers.logical_and(gt_iou_map <= 0.7, gt_iou_map > 0.3)
          u_mmask = fluid.layers.cast(x=u_mmask, dtype=DATATYPE)
          u_lmask = fluid.layers.logical_and(gt_iou_map <= 0.3, gt_iou_map >= 0.)
          u_lmask = fluid.layers.cast(x=u_lmask, dtype=DATATYPE)
          u_lmask = fluid.layers.elementwise_mul(u_lmask, mask)
  
          num_h = fluid.layers.cast(
              fluid.layers.reduce_sum(u_hmask), dtype=DATATYPE)
          num_m = fluid.layers.cast(
              fluid.layers.reduce_sum(u_mmask), dtype=DATATYPE)
          num_l = fluid.layers.cast(
              fluid.layers.reduce_sum(u_lmask), dtype=DATATYPE)
  
          r_m = num_h / num_m
          u_smmask = fluid.layers.uniform_random(
              shape=[gt_iou_map.shape[1], gt_iou_map.shape[2]],
              dtype=DATATYPE,
              min=0.0,
              max=1.0)
          u_smmask = fluid.layers.elementwise_mul(u_mmask, u_smmask)
          u_smmask = fluid.layers.cast(x=(u_smmask > (1. - r_m)), dtype=DATATYPE)
  
          r_l = num_h / num_l
          u_slmask = fluid.layers.uniform_random(
              shape=[gt_iou_map.shape[1], gt_iou_map.shape[2]],
              dtype=DATATYPE,
              min=0.0,
              max=1.0)
          u_slmask = fluid.layers.elementwise_mul(u_lmask, u_slmask)
          u_slmask = fluid.layers.cast(x=(u_slmask > (1. - r_l)), dtype=DATATYPE)
  
          weights = u_hmask + u_smmask + u_slmask
          weights.stop_gradient = True
          loss = fluid.layers.square_error_cost(pred_score, gt_iou_map)
          loss = fluid.layers.elementwise_mul(loss, weights)
          loss = 0.5 * fluid.layers.reduce_sum(loss) / fluid.layers.reduce_sum(
              weights)
  
          return loss
  
      def pem_cls_loss_func(pred_score, gt_iou_map, mask):
          gt_iou_map = fluid.layers.elementwise_mul(gt_iou_map, mask)
          gt_iou_map.stop_gradient = True
          pmask = fluid.layers.cast(x=(gt_iou_map > 0.9), dtype=DATATYPE)
          nmask = fluid.layers.cast(x=(gt_iou_map <= 0.9), dtype=DATATYPE)
          nmask = fluid.layers.elementwise_mul(nmask, mask)
  
          num_positive = fluid.layers.reduce_sum(pmask)
          num_entries = num_positive + fluid.layers.reduce_sum(nmask)
          ratio = num_entries / num_positive
          coef_0 = 0.5 * ratio / (ratio - 1)
          coef_1 = 0.5 * ratio
          epsilon = 0.000001
          loss_pos = fluid.layers.elementwise_mul(
              fluid.layers.log(pred_score + epsilon), pmask)
          loss_pos = coef_1 * fluid.layers.reduce_sum(loss_pos)
          loss_neg = fluid.layers.elementwise_mul(
              fluid.layers.log(1.0 - pred_score + epsilon), nmask)
          loss_neg = coef_0 * fluid.layers.reduce_sum(loss_neg)
          loss = -1 * (loss_pos + loss_neg) / num_entries
          return loss
  
      pred_bm_reg = fluid.layers.squeeze(
          fluid.layers.slice(
              pred_bm, axes=[1], starts=[0], ends=[1]), axes=[1])
      pred_bm_cls = fluid.layers.squeeze(
          fluid.layers.slice(
              pred_bm, axes=[1], starts=[1], ends=[2]), axes=[1])
  
      bm_mask = _get_mask(cfg)
  
      pem_reg_loss = pem_reg_loss_func(pred_bm_reg, gt_iou_map, bm_mask)
      pem_cls_loss = pem_cls_loss_func(pred_bm_cls, gt_iou_map, bm_mask)
  
      tem_loss = tem_loss_func(pred_start, pred_end, gt_start, gt_end)
  
      loss = tem_loss + 10 * pem_reg_loss + pem_cls_loss
      return loss, tem_loss, pem_reg_loss, pem_cls_loss
  
apis: numpy.sum numpy.argpartition numpy.argmax numpy.arange numpy.average numpy.mean
apis: numpy.array