bdlb/diabetic_retinopathy_diagnosis/benchmark.py
def _evaluate_metric(
      y_true: np.ndarray,
      y_pred: np.ndarray,
      y_uncertainty: np.ndarray,
      fractions: Sequence[float],
      metric_fn: Callable[[np.ndarray, np.ndarray], float],
      name=None,
  ) -> pd.DataFrame:
    """Evaluate model predictive distribution on `metric_fn` at data retain
    `fractions`.

    Args:
      y_true: `numpy.ndarray`, the ground truth labels, with shape [N].
      y_pred: `numpy.ndarray`, the model predictions, with shape [N].
      y_uncertainty: `numpy.ndarray`, the model uncertainties,
        with shape [N].
      fractions: `iterable`, the percentages of data to retain for
        calculating `metric_fn`.
      metric_fn: `lambda(y_true, y_pred) -> float`, a metric
        function that provides a score given ground truths
        and predictions.
      name: (optional) `str`, the name of the method.

    Returns:
      A `pandas.DataFrame` with columns ["retained_data", "mean", "std"],
      that summarizes the scores at different data retained fractions.
    """

    N = y_true.shape[0]

    # Sorts indexes by ascending uncertainty
    I_uncertainties = np.argsort(y_uncertainty)

    # Score containers
    mean = np.empty_like(fractions)
    # TODO(filangel): do bootstrap sampling and estimate standard error
    std = np.zeros_like(fractions)

    for i, frac in enumerate(fractions):
      # Keep only the %-frac of lowest uncertainties
      I = np.zeros(N, dtype=bool)
      I[I_uncertainties[:int(N * frac)]] = True
      mean[i] = metric_fn(y_true[I], y_pred[I])

    # Store
    df = pd.DataFrame(dict(retained_data=fractions, mean=mean, std=std))
    df.name = name

    return df
Evaluate model predictive distribution on `metric_fn` at data retain
    `fractions`.

    Args:
      y_true: `numpy.ndarray`, the ground truth labels, with shape [N].
      y_pred: `numpy.ndarray`, the model predictions, with shape [N].
      y_uncertainty: `numpy.ndarray`, the model uncertainties,
        with shape [N].
      fractions: `iterable`, the percentages of data to retain for
        calculating `metric_fn`.
      metric_fn: `lambda(y_true, y_pred) -> float`, a metric
        function that provides a score given ground truths
        and predictions.
      name: (optional) `str`, the name of the method.

    Returns:
      A `pandas.DataFrame` with columns ["retained_data", "mean", "std"],
      that summarizes the scores at different data retained fractions.
    
Evaluate model predictive distribution on `metric_fn` at data retain
`fractions`.
Evaluate model predictive distribution on ` metric_fn ` at data retain ` fractions ` .
def _evaluate_metric(
      y_true: np.ndarray,
      y_pred: np.ndarray,
      y_uncertainty: np.ndarray,
      fractions: Sequence[float],
      metric_fn: Callable[[np.ndarray, np.ndarray], float],
      name=None,
  ) -> pd.DataFrame:
    N = y_true.shape[0]
    I_uncertainties = np.argsort(y_uncertainty)
    mean = np.empty_like(fractions)
    std = np.zeros_like(fractions)
    for i, frac in enumerate(fractions):
      I = np.zeros(N, dtype=bool)
      I[I_uncertainties[:int(N * frac)]] = True
      mean[i] = metric_fn(y_true[I], y_pred[I])
    df = pd.DataFrame(dict(retained_data=fractions, mean=mean, std=std))
    df.name = name
    return df
def _evaluate_metric ( y_true : np . ndarray , y_pred : np . ndarray , y_uncertainty : np . ndarray , fractions : Sequence [ float ] , metric_fn : Callable [ [ np . ndarray , np . ndarray ] , float ] , name = None , ) -> pd . DataFrame : N = y_true . shape [ 0 ] I_uncertainties = np . argsort ( y_uncertainty ) mean = np . empty_like ( fractions ) std = np . zeros_like ( fractions ) for i , frac in enumerate ( fractions ) : I = np . zeros ( N , dtype = bool ) I [ I_uncertainties [ : int ( N * frac ) ] ] = True mean [ i ] = metric_fn ( y_true [ I ] , y_pred [ I ] ) df = pd . DataFrame ( dict ( retained_data = fractions , mean = mean , std = std ) ) df . name = name return df
Evaluate model predictive distribution on `metric_fn` at data retain
`fractions`.
Evaluate model predictive distribution on ` metric_fn ` at data retain ` fractions ` .
"""Evaluate model predictive distribution on `metric_fn` at data retain
    `fractions`.

    Args:
      y_true: `numpy.ndarray`, the ground truth labels, with shape [N].
      y_pred: `numpy.ndarray`, the model predictions, with shape [N].
      y_uncertainty: `numpy.ndarray`, the model uncertainties,
        with shape [N].
      fractions: `iterable`, the percentages of data to retain for
        calculating `metric_fn`.
      metric_fn: `lambda(y_true, y_pred) -> float`, a metric
        function that provides a score given ground truths
        and predictions.
      name: (optional) `str`, the name of the method.

    Returns:
      A `pandas.DataFrame` with columns ["retained_data", "mean", "std"],
      that summarizes the scores at different data retained fractions.
    """ # Sorts indexes by ascending uncertainty # Score containers # TODO(filangel): do bootstrap sampling and estimate standard error # Keep only the %-frac of lowest uncertainties # Store
A `pandas.DataFrame` with columns ["retained_data", "mean", "std"],
that summarizes the scores at different data retained fractions.
A ` pandas . DataFrame ` with columns [ " retained_data " " mean " " std " ] that summarizes the scores at different data retained fractions .
y_true
np.ndarray
`numpy.ndarray`, the ground truth labels, with shape [N].
` numpy . ndarray ` the ground truth labels with shape [ N ] .
y_pred
np.ndarray
`numpy.ndarray`, the model predictions, with shape [N].
` numpy . ndarray ` the model predictions with shape [ N ] .
y_uncertainty
np.ndarray
`numpy.ndarray`, the model uncertainties,
with shape [N].
` numpy . ndarray ` the model uncertainties with shape [ N ] .
fractions
Sequence[float]
`iterable`, the percentages of data to retain for
calculating `metric_fn`.
` iterable ` the percentages of data to retain for calculating ` metric_fn ` .
metric_fn
Callable[[np.ndarray, np.ndarray], float]
`lambda(y_true, y_pred) -> float`, a metric
function that provides a score given ground truths
and predictions.
` lambda ( y_true y_pred ) - > float ` a metric function that provides a score given ground truths and predictions .
name
(optional) `str`, the name of the method.
( optional ) ` str ` the name of the method .