text: # BSD 2-CLAUSE LICENSE
  
  # Redistribution and use in source and binary forms, with or without modification,
  # are permitted provided that the following conditions are met:
  
  # Redistributions of source code must retain the above copyright notice, this
  # list of conditions and the following disclaimer.
  # Redistributions in binary form must reproduce the above copyright notice,
  # this list of conditions and the following disclaimer in the documentation
  # and/or other materials provided with the distribution.
  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
  # #ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  # original author: Reza Hosseini, Albert Chen
  """Evaluation functions.
  Valid input processing is done within these evaluation functions
  so they can be called from anywhere without error checks.
  """
  
  import math
  import warnings
  from collections import namedtuple
  from enum import Enum
  from functools import partial
  
  import numpy as np
  import pandas as pd
  from scipy.stats import pearsonr
  from sklearn.metrics import mean_absolute_error
  from sklearn.metrics import mean_squared_error
  from sklearn.metrics import median_absolute_error
  from sklearn.metrics import r2_score
  
  from greykite.common.constants import ACTUAL_COL
  from greykite.common.constants import COVERAGE_VS_INTENDED_DIFF
  from greykite.common.constants import LOWER_BAND_COVERAGE
  from greykite.common.constants import PREDICTED_COL
  from greykite.common.constants import PREDICTED_LOWER_COL
  from greykite.common.constants import PREDICTED_UPPER_COL
  from greykite.common.constants import PREDICTION_BAND_COVERAGE
  from greykite.common.constants import PREDICTION_BAND_WIDTH
  from greykite.common.constants import UPPER_BAND_COVERAGE
  from greykite.common.logging import LoggingLevelEnum
  from greykite.common.logging import log_message
  
  
  def all_equal_length(*arrays):
      """Checks whether input arrays have equal length.
  
      :param arrays: one or more lists/numpy arrays/pd.Series
      :return: true if lengths are all equal, otherwise false
      """
      length = None
      for arr in arrays:
          try:
              if length is not None and len(arr) != length:
                  return False
              length = len(arr)
          except TypeError:  # continue if `arr` has no len method. constants and None are excluded from length check
              pass
      return True
  
  
  def valid_elements_for_evaluation(reference_array, *arrays):
      """Keeps finite elements from reference_array, and corresponding elements in *arrays.
  
      :param reference_array: np.array, pd.Series, or list
      :param arrays: additional array-like objects of the same length
          can be None or numeric, in which case original value is returned
      :return: list of numpy arrays with valid indices [reference_array, *arrays]
      """
      if not all_equal_length(reference_array, *arrays):
          raise Exception("length of arrays do not match")
  
      reference_array = np.array(reference_array)
      # Finds the indices of finite elements in reference array
      keep = np.isfinite(reference_array)
      num_remaining = keep.sum()
      num_removed = reference_array.shape[0] - num_remaining
      if num_remaining == 0:
          warnings.warn(f"There are 0 non-null elements for evaluation.")
      if num_removed > 0:
          warnings.warn(f"{num_removed} value(s) in y_true were NA or infinite and are omitted in error calc.")
  
      # Keeps these indices in all arrays. Leaves float, int, and None as-is
      return [reference_array[keep]] + [np.array(array)[keep] if (array is not None
                                                                  and not isinstance(array, float)
                                                                  and not isinstance(array, int))
                                        else array
                                        for array in arrays]
  
  
  def aggregate_array(ts_values, agg_periods=7, agg_func=np.sum):
      """Aggregates input array.
      Divides array from left to right into bins of size agg_periods, and applies agg_func to each block.
      Drops records from the left if needed to ensure all bins are full.
  
      :param ts_values: list, np.array, or pd.Series to aggregate
      :param agg_periods: number of periods to combine in aggregation
      :param agg_func: aggregation function, e.g. np.max, np.sum. Must take an array and returns a number
      :return: array, aggregated so that every agg_periods periods are combined into one
  
      Examples:
      >>> aggregate_array([1.0, 2.0, 3.0, 4.0], 2, np.sum)
      array([3., 7.])
      >>> aggregate_array(pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]), 2, np.sum)
      array([5., 9.])
      >>> aggregate_array(np.array([1.0, 2.0, 3.0, 4.0, 5.0]), 2, np.max)
      array([3., 5.])
      """
      ts_values = np.array(ts_values)
      n_periods = len(ts_values)
  
      drop_first_periods = n_periods % agg_periods  # drop these periods from the front, to ensure all bins are full
      if drop_first_periods == n_periods:
          drop_first_periods = 0
          warnings.warn(f"Requested agg_periods={agg_periods}, but there are only {n_periods}. Using all for aggregation")
      elif drop_first_periods > 0:
          log_message(f"Requested agg_periods={agg_periods} for data of length {n_periods}. Dropping first"
                      f" {drop_first_periods} records before aggregation", LoggingLevelEnum.INFO)
  
      # creates dummy time index for aggregation
      dates = pd.date_range("2018-01-01", periods=n_periods - drop_first_periods, freq="1D")
      ts = pd.Series(ts_values[drop_first_periods:], index=dates)
      aggregated_array = ts.resample(f"{agg_periods}D", closed="left") \
          .agg(lambda x: agg_func(x)) \
          .values
      return aggregated_array
  
  
  def add_preaggregation_to_scorer(score_func, agg_periods=7, agg_func=np.sum):
      """Takes a scorer and returns a scorer that pre-aggregates input.
  
      :param score_func: function that maps two arrays to a number. E.g. (y_true, y_pred) -> error
      :param agg_periods: number of periods to combine in aggregation
      :param agg_func: function that takes an array and returns a number, e.g. np.max, np.sum. This function should
          handle np.nan without error
      :return: scorer that applies agg_func to data before calling score_func
  
      For example, if have daily data and want to evaluate MSE on weekly totals, the appropriate scorer is:
          add_preaggregation_to_scorer(mean_squared_error, agg_periods=7, agg_func=np.max)
      """
      def score_func_preagg(y_true, y_pred, **kwargs):
          y_true_agg = aggregate_array(y_true, agg_periods, agg_func)
          y_pred_agg = aggregate_array(y_pred, agg_periods, agg_func)
          return score_func(y_true_agg, y_pred_agg, **kwargs)
      return score_func_preagg
  
  
  def add_finite_filter_to_scorer(score_func):
      """Takes a scorer and returns a scorer that ignores NA / infinite elements in y_true.
  
      sklearn scorers (and others) don't handle arrays with 0 length. In that case, return None
  
      :param score_func: function that maps two arrays to a number. E.g. (y_true, y_pred) -> error
      :return: scorer that drops records where y_true is not finite
      """
      def score_func_finite(y_true, y_pred, **kwargs):
          y_true, y_pred = valid_elements_for_evaluation(y_true, y_pred)
          if len(y_true) == 0:  # returns None if there are no elements
              return None
          return score_func(y_true, y_pred, **kwargs)
      return score_func_finite
  
  
  def r2_null_model_score(
          y_true,
          y_pred,
          y_pred_null=None,
          y_train=None,
          loss_func=mean_squared_error):
      """Calculates improvement in the loss function compared
      to the predictions of a null model. Can be used to evaluate
      model quality with respect to a simple baseline model.
  
      The score is defined as::
  
          R2_null_model_score = 1.0 - loss_func(y_true, y_pred) / loss_func(y_true, y_pred_null)
  
      Parameters
      ----------
      y_true : `list` [`float`] or `numpy.array`
          Observed response (usually on a test set).
      y_pred : `list` [`float`] or `numpy.array`
          Model predictions (usually on a test set).
      y_pred_null : `list` [`float`] or `numpy.array` or None
          A baseline prediction model to compare against.
          If None, derived from ``y_train`` or ``y_true``.
      y_train : `list` [`float`] or `numpy.array` or None
          Response values in the training data.
          If ``y_pred_null`` is None, then ``y_pred_null`` is set to the mean of ``y_train``.
          If ``y_train`` is also None, then ``y_pred_null`` is set to the mean of ``y_true``.
      loss_func : callable, default `sklearn.metrics.mean_squared_error`
          The error loss function with signature (true_values, predicted_values).
  
      Returns
      -------
      r2_null_model : `float`
          A value within (-\\infty, 1.0]. Higher scores are better.
          Can be interpreted as the improvement in the loss function
          compared to the predictions of the null model.
          For example, a score of 0.74 means the loss is 74% lower
          than for the null model.
  
      Notes
      -----
      There is a connection between ``R2_null_model_score`` and ``R2``.
      ``R2_null_model_score`` can be interpreted as the additional improvement in the
      coefficient of determination (i.e. ``R2``, see `sklearn.metrics.r2_score`) with
      respect to a null model.
  
      Under the default settings of this function, where ``loss_func`` is
      mean squared error and ``y_pred_null`` is the average of
      ``y_true``, the scores are equivalent::
  
          # simplified definition of R2_score, where SSE is sum of squared error
          y_true_avg = np.repeat(np.average(y_true), y_true.shape[0])
          R2_score := 1.0 - SSE(y_true, y_pred) / SSE(y_true, y_true_avg)
          R2_score := 1.0 - MSE(y_true, y_pred) / VAR(y_true)  # equivalent definition
  
          r2_null_model_score(y_true, y_pred) == r2_score(y_true, y_pred)
  
      ``r2_score`` is 0 if simply predicting the mean (y_pred = y_true_avg).
  
      If ``y_pred_null`` is passed, and if ``loss_func`` is mean squared error
      and ``y_true`` has nonzero variance, this function measures how much "r2_score of the
      predictions (``y_pred``)" closes the gap between "r2_score of the null model (``y_pred``)"
      and the "r2_score of the best possible model (``y_true``)", which is 1.0::
  
          R2_pred = r2_score(y_true, y_pred)       # R2 of predictions
          R2_null = r2_score(y_pred_null, y_pred)  # R2 of null model
          r2_null_model_score(y_true, y_pred, y_pred_null) == (R2_pred - R2_null) / (1.0 - R2_null)
  
      When ``y_pred_null=y_true_avg``, ``R2_null`` is 0 and this reduces to the formula above.
  
      Summary (for ``loss_func=mean_squared_error``):
  
          - If ``R2_null>0`` (good null model), then ``R2_null_model_score < R2_score``
          - If ``R2_null=0`` (uninformative null model), then ``R2_null_model_score = R2_score``
          - If ``R2_null<0`` (poor null model), then ``R2_null_model_score > R2_score``
  
      For other loss functions, ``r2_null_model_score`` has the same connection to pseudo R2.
      """
      y_true, y_pred, y_train, y_pred_null = valid_elements_for_evaluation(y_true, y_pred, y_train, y_pred_null)
      r2_null_model = None
  
      if len(y_true) > 0:
          # determines null model
          if y_pred_null is None and y_train is not None:  # from training data
              y_pred_null = np.repeat(y_train.mean(), len(y_true))
          elif y_pred_null is None:  # from test data
              y_pred_null = np.repeat(y_true.mean(), len(y_true))
          elif isinstance(y_pred_null, float) or isinstance(y_pred_null, int):  # constant null model
              y_pred_null = np.repeat(y_pred_null, len(y_true))
          # otherwise, y_pred_null is an array, used directly
  
          model_loss = loss_func(y_true, y_pred)
          null_loss = loss_func(y_true, y_pred_null)
  
          if null_loss > 0.0:
              r2_null_model = 1.0 - (model_loss / null_loss)
      return r2_null_model
  
  
  def calc_pred_err(y_true, y_pred):
      """Calculates the basic error measures in
      `~greykite.common.evaluation.EvaluationMetricEnum`
      and returns them in a dictionary.
  
      Parameters
      ----------
      y_true : `list` [`float`] or `numpy.array`
          Observed values.
      y_pred : `list` [`float`] or `numpy.array`
          Model predictions.
  
      Returns
      -------
      error : `dict` [`str`, `float` or None]
          Dictionary mapping
          `~greykite.common.evaluation.EvaluationMetricEnum`
          metric names to their values for the given ``y_true``
          and ``y_pred``.
  
          The dictionary is empty is there are no finite elements
          in ``y_true``.
      """
      y_true, y_pred = valid_elements_for_evaluation(y_true, y_pred)
      error = {}
  
      if len(y_true) > 0:
          for enum in EvaluationMetricEnum:
              metric_name = enum.get_metric_name()
              metric_func = enum.get_metric_func()
              error.update({metric_name: metric_func(y_true, y_pred)})
      return error
  
  
  @add_finite_filter_to_scorer
  def mean_absolute_percent_error(y_true, y_pred):
      """Calculates mean absolute percentage error.
  
      :param y_true: observed values given in a list (or numpy array)
      :param y_pred: predicted values given in a list (or numpy array)
      :return: mean absolute percent error
      """
      smallest_denominator = np.min(np.abs(y_true))
      if smallest_denominator == 0:
          warnings.warn("y_true contains 0. MAPE is undefined.")
          return None
      elif smallest_denominator < 1e-8:
          warnings.warn("y_true contains very small values. MAPE is likely highly volatile.")
      return 100 * (np.abs(y_true - y_pred) / np.abs(y_true)).mean()
  
  
  @add_finite_filter_to_scorer
  def median_absolute_percent_error(y_true, y_pred):
      """Calculates median absolute percentage error.
  
      :param y_true: observed values given in a list (or numpy array)
      :param y_pred: predicted values given in a list (or numpy array)
      :return: median absolute percent error
      """
      smallest_denominator = np.min(np.abs(y_true))
      if smallest_denominator == 0:
          warnings.warn("y_true contains 0. MedAPE is undefined.")
          return None
      elif smallest_denominator < 1e-8:
          num_small = len(y_true[y_true < 1e-8])
          if num_small > (len(y_true) - 1) // 2:
              # if too many very small values, median is affected
              warnings.warn("y_true contains very small values. MedAPE is likely highly volatile.")
      return 100 * np.median((np.abs(y_true - y_pred) / np.abs(y_true)))
  
  
  @add_finite_filter_to_scorer
  def symmetric_mean_absolute_percent_error(y_true, y_pred):
      """Calculates symmetric mean absolute percentage error
      Note that we do not include a factor of 2 in the denominator, so the range is 0% to 100%.
  
      :param y_true: observed values given in a list (or numpy array)
      :param y_pred: predicted values given in a list (or numpy array)
      :return: symmetric mean absolute percent error
      """
      smallest_denominator = np.min(np.abs(y_true) + np.abs(y_pred))
      if smallest_denominator == 0:
          warnings.warn("denominator contains 0. sMAPE is undefined.")
          return None
      elif smallest_denominator < 1e-8:
          warnings.warn("denominator contains very small values. sMAPE is likely highly volatile.")
      return 100 * (np.abs(y_true - y_pred) / (np.abs(y_true) + np.abs(y_pred))).mean()
  
  
  @add_finite_filter_to_scorer
  def root_mean_squared_error(y_true, y_pred):
      """Calculates root mean square error.
  
      :param y_true: observed values given in a list (or numpy array)
      :param y_pred: predicted values given in a list (or numpy array)
      :return: mean absolute percent error
      """
      return math.sqrt(mean_squared_error(y_true, y_pred))
  
  
  @add_finite_filter_to_scorer
  def correlation(y_true, y_pred):
      """Calculates correlation.
  
      :param y_true: observed values given in a list (or numpy array)
      :param y_pred: predicted values given in a list (or numpy array)
      :return: correlation
      """
      if np.unique(y_true).size == 1:
          warnings.warn("y_true is constant. Correlation is not defined.")
          return None
      elif np.unique(y_pred).size == 1:
          warnings.warn("y_pred is constant. Correlation is not defined.")
          return None
      else:
          return pearsonr(y_true, y_pred)[0]
  
  
  @add_finite_filter_to_scorer
  def quantile_loss(y_true, y_pred, q: float = 0.95):
      """Calculates the quantile (pinball) loss with quantile q of y_true and y_pred.
  
      :param y_true: one column of true values.
      :type y_true: list or numpy array
      :param y_pred: one column of predicted values.
      :type y_pred: list or numpy array.
      :param q: the quantile.
      :type q: float
      :return: quantile loss.
      :returns: float
      """
      return np.where(y_true < y_pred, (1 - q) * (y_pred - y_true), q * (y_true - y_pred)).mean()
  
  
  def quantile_loss_q(q):
      """Returns quantile loss function for the specified quantile"""
      def quantile_loss_wrapper(y_true, y_pred):
          return quantile_loss(y_true, y_pred, q=q)
      return quantile_loss_wrapper
  
  
  def fraction_within_bands(observed, lower, upper):
      """Computes the fraction of observed values between lower and upper.
  
      :param observed: pd.Series or np.array, numeric, observed values
      :param lower: pd.Series or np.array, numeric, lower bound
      :param upper: pd.Series or np.array, numeric, upper bound
      :return: float between 0.0 and 1.0
      """
      observed, lower, upper = valid_elements_for_evaluation(observed, lower, upper)
      num_reversed = np.count_nonzero(upper < lower)
      if num_reversed > 0:
          warnings.warn(f"{num_reversed} of {len(observed)} upper bound values are smaller than the lower bound")
      return np.count_nonzero((observed > lower) & (observed < upper)) / len(observed) if len(observed) > 0 else None
  
  
  @add_finite_filter_to_scorer
  def fraction_outside_tolerance(y_true, y_pred, rtol=0.05):
      """Returns the fraction of predicted values whose relative difference
      from the true value is strictly greater than ``rtol``.
  
      Parameters
      ----------
      y_true : `list` or `numpy.array`
          True values.
      y_pred : `list` or `numpy.array`
          Predicted values.
      rtol : `float`, default 0.05
          Relative error tolerance.
          For example, 0.05 allows for 5% relative error.
  
      Returns
      -------
      frac_outside_tolerance : float
          Fraction of values outside tolerance. (0.0 to 1.0)
          A value is outside tolerance if `numpy.isclose`
          with the specified ``rtol`` and ``atol=0.0`` returns False.
      """
      return np.mean(~np.isclose(y_pred, y_true, rtol=rtol, atol=0.0))
  
  
  def prediction_band_width(observed, lower, upper):
      """Computes the prediction band width, expressed as a % relative to observed.
      Width is defined as average ratio of (upper-lower)/observed.
  
      :param observed: pd.Series or np.array, numeric, observed values
      :param lower: pd.Series or np.array, numeric, lower bound
      :param upper: pd.Series or np.array, numeric, upper bound
      :return: float, average percentage width
      """
      observed, lower, upper = valid_elements_for_evaluation(observed, lower, upper)
      observed = np.abs(observed)
      num_reversed = np.count_nonzero(upper < lower)
      if num_reversed > 0:
          warnings.warn(f"{num_reversed} of {len(observed)} upper bound values are smaller than the lower bound")
      # if there are 0s in observed, relative size is undefined
      return 100.0 * np.mean(np.abs(upper - lower) / observed) if len(observed) > 0 and observed.min() > 0 else None
  
  
  def calc_pred_coverage(observed, predicted, lower, upper, coverage):
      """Calculates the prediction coverages:
     prediction band width, prediction band coverage etc.
  
      :param observed: pd.Series or np.array, numeric, observed values
      :param predicted: pd.Series or np.array, numeric, predicted values
      :param lower: pd.Series or np.array, numeric, lower bound
      :param upper: pd.Series or np.array, numeric, upper bound
      :param coverage: float, intended coverage of the prediction bands (0.0 to 1.0)
      :return: prediction band width, prediction band coverage etc.
      """
      observed, predicted, lower, upper = valid_elements_for_evaluation(
          observed,
          predicted,
          lower,
          upper)
      metrics = {}
  
      if len(observed) > 0:
          # relative size of prediction bands vs actual, as a percent
          enum = ValidationMetricEnum.BAND_WIDTH
          metric_func = enum.get_metric_func()
          metrics.update({PREDICTION_BAND_WIDTH: metric_func(observed, lower, upper)})
  
          enum = ValidationMetricEnum.BAND_COVERAGE
          metric_func = enum.get_metric_func()
          # fraction of observations within the bands
          metrics.update({PREDICTION_BAND_COVERAGE: metric_func(observed, lower, upper)})
          # fraction of observations within the lower band
          metrics.update({LOWER_BAND_COVERAGE: metric_func(observed, lower, predicted)})
          # fraction of observations within the upper band
          metrics.update({UPPER_BAND_COVERAGE: metric_func(observed, predicted, upper)})
          # difference between actual and intended coverage
          metrics.update({COVERAGE_VS_INTENDED_DIFF: (metrics[PREDICTION_BAND_COVERAGE] - coverage)})
      return metrics
  
  
  def elementwise_residual(true_val, pred_val):
      """The residual between a single true and predicted value.
  
      Parameters
      ----------
      true_val : float
          True value.
      pred_val : float
          Predicted value.
  
      Returns
      -------
      residual : float
          The residual, true minus predicted
      """
      return true_val - pred_val
  
  
  def elementwise_absolute_error(true_val, pred_val):
      """The absolute error between a single true and predicted value.
  
      Parameters
      ----------
      true_val : float
          True value.
      pred_val : float
          Predicted value.
  
      Returns
      -------
      residual : float
          Absolute error, |true_val - pred_val|
      """
      return abs(true_val - pred_val)
  
  
  def elementwise_squared_error(true_val, pred_val):
      """The absolute error between a single true and predicted value.
  
      Parameters
      ----------
      true_val : float
          True value.
      pred_val : float
          Predicted value.
  
      Returns
      -------
      residual : float
          Squared error, (true_val - pred_val)^2
      """
      return (true_val - pred_val)**2
  
  
  def elementwise_absolute_percent_error(true_val, pred_val):
      """The absolute percent error between a single true and predicted value.
  
      Parameters
      ----------
      true_val : float
          True value.
      pred_val : float
          Predicted value.
  
      Returns
      -------
      percent_error : float
          Percent error, pred_val / true_val - 1
      """
      if true_val == 0:
          warnings.warn("true_val is 0. Percent error is undefined.")
          return np.nan
      elif true_val < 1e-8:
          warnings.warn("true_val is less than 1e-8. Percent error is very likely highly volatile.")
      return 100 * abs(true_val - pred_val) / abs(true_val)
  
  
  def elementwise_quantile(true_val, pred_val, q):
      """The quantile loss between a single true and predicted value.
  
      Parameters
      ----------
      true_val : float
          True value.
      pred_val : float
          Predicted value.
  
      Returns
      -------
      quantile_loss : float
          Quantile loss, absolute error weighed by ``q``
          for underpredictions and ``1-q`` for overpredictions.
      """
      weight = 1 - q if true_val < pred_val else q
      return weight * abs(true_val - pred_val)
  
  
  def elementwise_outside_tolerance(true_val, pred_val, rtol=0.05):
      """Whether the relative difference between ``pred_val`` and
      ``true_val`` is strictly greater than ``rtol``.
  
      Parameters
      ----------
      true_val : float
          True value.
      pred_val : float
          Predicted value.
      rtol : float, default 0.05
          Relative error tolerance.
          For example, 0.05 allows for 5% relative error.
  
      Returns
      -------
      is_outside_tolerance : float
          1.0 if the error is outside tolerance, else 0.0.
          A value is outside tolerance if `numpy.isclose`
          with the specified ``rtol`` and ``atol=0.0`` returns False.
  
      See Also
      --------
  
      """
      return 0.0 if np.isclose(pred_val, true_val, rtol=rtol, atol=0.0) else 1.0
  
  
  def elementwise_within_bands(true_val, lower_val, upper_val):
      """Whether ``true_val`` is strictly between ``lower_val`` and ``upper_val``.
  
      Parameters
      ----------
      true_val : float
          True value.
      lower_val : float
          Lower bound.
      upper_val : float
          Upper bound.
  
      Returns
      -------
      is_within_band : float
          1.0 if error is strictly within the limits, else 0.0
      """
      return 1.0 if lower_val < true_val < upper_val else 0.0
  
  
  class ElementwiseEvaluationMetricEnum(Enum):
      """Evaluation metrics for a single element.
      This is the function computed on each row, before aggregation across records.
      The aggregation function can be mean, median, quantile, max, sqrt of the mean, etc.
  
      For example, AbsoluteError followed by mean aggregation gives MeanAbsoluteError.
      EvaluationMetricEnum is more efficient if it can be used directly.
      """
      ElementwiseEvaluationMetric = namedtuple("ElementwiseEvaluationMetric", "func, name, args")
      """
      The elementwise evaluation function and associated metadata.
  
          ``"func"`` : `callable`
              Elementwise function.
          ``"name"`` : `str`
              Short name for the metric.
          ``"args"`` : `list` [`str`]
              Description of function arguments.
              e.g. [actual, predicted]
      """
      Residual = ElementwiseEvaluationMetric(
          elementwise_residual,
          "residual",
          [ACTUAL_COL, PREDICTED_COL])
      """Residual, true-pred"""
      AbsoluteError = ElementwiseEvaluationMetric(
          elementwise_absolute_error,
          "absolute_error",
          [ACTUAL_COL, PREDICTED_COL])
      """Absolute error, abs(true-pred)"""
      SquaredError = ElementwiseEvaluationMetric(
          elementwise_squared_error,
          "squared_error",
          [ACTUAL_COL, PREDICTED_COL])
      """Squared error, (true-pred)^2"""
      AbsolutePercentError = ElementwiseEvaluationMetric(
          elementwise_absolute_percent_error,
          "absolute_percent_error",
          [ACTUAL_COL, PREDICTED_COL])
      """Percent error, abs(true-pred)/abs(true)"""
      Quantile80 = ElementwiseEvaluationMetric(
          partial(elementwise_quantile, q=0.80),
          "quantile_loss_80",
          [ACTUAL_COL, PREDICTED_COL])
      """Quantile loss with q=0.80"""
      Quantile90 = ElementwiseEvaluationMetric(
          partial(elementwise_quantile, q=0.90),
          "quantile_loss_90",
          [ACTUAL_COL, PREDICTED_COL])
      """Quantile loss with q=0.90"""
      Quantile95 = ElementwiseEvaluationMetric(
          partial(elementwise_quantile, q=0.95),
          "quantile_loss_95",
          [ACTUAL_COL, PREDICTED_COL])
      """Quantile loss with q=0.95"""
      Quantile99 = ElementwiseEvaluationMetric(
          partial(elementwise_quantile, q=0.99),
          "quantile_loss_99",
          [ACTUAL_COL, PREDICTED_COL])
      """Quantile loss with q=0.99"""
      OutsideTolerance5 = ElementwiseEvaluationMetric(
          partial(elementwise_outside_tolerance, rtol=0.05),
          "outside_tolerance_5p",
          [ACTUAL_COL, PREDICTED_COL])
      """Whether the predicted value deviates more than 5% from the true value"""
      Coverage = ElementwiseEvaluationMetric(
          elementwise_within_bands,
          "coverage",
          [ACTUAL_COL, PREDICTED_LOWER_COL, PREDICTED_UPPER_COL])
      """Whether the actual value is within the prediction interval"""
  
      def get_metric_func(self):
          """Returns the metric function"""
          return self.value.func
  
      def get_metric_name(self):
          """Returns the short name"""
          return self.value.name
  
      def get_metric_args(self):
          """Returns the expected argument list"""
          return self.value.args
  
  
  class EvaluationMetricEnum(Enum):
      """Valid evaluation metrics.
      The values tuple is ``(score_func: callable, greater_is_better: boolean, short_name: str)``
  
      ``add_finite_filter_to_scorer`` is added to the metrics that are directly imported from
      ``sklearn.metrics`` (e.g. ``mean_squared_error``) to ensure that the metric gets calculated
      even when inputs have missing values.
      """
      Correlation = (correlation, True, "CORR")
      """Pearson correlation coefficient between forecast and actuals. Higher is better."""
      CoefficientOfDetermination = (add_finite_filter_to_scorer(r2_score), True, "R2")
      """Coefficient of determination. See `sklearn.metrics.r2_score`. Higher is better. Equals `1.0 - mean_squared_error / variance(actuals)`."""
      MeanSquaredError = (add_finite_filter_to_scorer(mean_squared_error), False, "MSE")
      """Mean squared error, the average of squared differences,
      see `sklearn.metrics.mean_squared_error`."""
      RootMeanSquaredError = (root_mean_squared_error, False, "RMSE")
      """Root mean squared error, the square root of `sklearn.metrics.mean_squared_error`"""
      MeanAbsoluteError = (add_finite_filter_to_scorer(mean_absolute_error), False, "MAE")
      """Mean absolute error, average of absolute differences,
      see `sklearn.metrics.mean_absolute_error`."""
      MedianAbsoluteError = (add_finite_filter_to_scorer(median_absolute_error), False, "MedAE")
      """Median absolute error, median of absolute differences,
      see `sklearn.metrics.median_absolute_error`."""
      MeanAbsolutePercentError = (mean_absolute_percent_error, False, "MAPE")
      """Mean absolute percent error, error relative to actuals expressed as a %,
      see `wikipedia MAPE <https://en.wikipedia.org/wiki/Mean_absolute_percentage_error>`_."""
      MedianAbsolutePercentError = (median_absolute_percent_error, False, "MedAPE")
      """Median absolute percent error, median of error relative to actuals expressed as a %,
      a median version of the MeanAbsolutePercentError, less affected by extreme values."""
      SymmetricMeanAbsolutePercentError = (symmetric_mean_absolute_percent_error, False, "sMAPE")
      """Symmetric mean absolute percent error, error relative to (actuals+forecast) expressed as a %.
      Note that we do not include a factor of 2 in the denominator, so the range is 0% to 100%,
      see `wikipedia sMAPE <https://en.wikipedia.org/wiki/Symmetric_mean_absolute_percentage_error>`_."""
      Quantile80 = (quantile_loss_q(0.80), False, "Q80")
      """Quantile loss with q=0.80::
  
          np.where(y_true < y_pred, (1 - q) * (y_pred - y_true), q * (y_true - y_pred)).mean()
      """
      Quantile95 = (quantile_loss_q(0.95), False, "Q95")
      """Quantile loss with q=0.95::
  
          np.where(y_true < y_pred, (1 - q) * (y_pred - y_true), q * (y_true - y_pred)).mean()
      """
      Quantile99 = (quantile_loss_q(0.99), False, "Q99")
      """Quantile loss with q=0.99::
  
          np.where(y_true < y_pred, (1 - q) * (y_pred - y_true), q * (y_true - y_pred)).mean()
      """
      FractionOutsideTolerance1 = (partial(fraction_outside_tolerance, rtol=0.01), False, "OutsideTolerance1p")
      """Fraction of forecasted values that deviate more than 1% from the actual"""
      FractionOutsideTolerance2 = (partial(fraction_outside_tolerance, rtol=0.02), False, "OutsideTolerance2p")
      """Fraction of forecasted values that deviate more than 2% from the actual"""
      FractionOutsideTolerance3 = (partial(fraction_outside_tolerance, rtol=0.03), False, "OutsideTolerance3p")
      """Fraction of forecasted values that deviate more than 3% from the actual"""
      FractionOutsideTolerance4 = (partial(fraction_outside_tolerance, rtol=0.04), False, "OutsideTolerance4p")
      """Fraction of forecasted values that deviate more than 4% from the actual"""
      FractionOutsideTolerance5 = (partial(fraction_outside_tolerance, rtol=0.05), False, "OutsideTolerance5p")
      """Fraction of forecasted values that deviate more than 5% from the actual"""
  
      def get_metric_func(self):
          """Returns the metric function"""
          return self.value[0]
  
      def get_metric_greater_is_better(self):
          """Returns the greater_is_better boolean"""
          return self.value[1]
  
      def get_metric_name(self):
          """Returns the short name"""
          return self.value[2]
  
  
  class ValidationMetricEnum(Enum):
      """Valid diagnostic metrics.
      The values tuple is ``(score_func: callable, greater_is_better: boolean)``
      """
      BAND_WIDTH = (prediction_band_width, False)
      BAND_COVERAGE = (fraction_within_bands, True)
  
      def get_metric_func(self):
          return self.value[0]
  
      def get_metric_greater_is_better(self):
          return self.value[1]
  