text: ################################################################################
  # Name:         Scoring Program
  # Author:       Zhengying Liu, Isabelle Guyon, Adrien Pavao, Zhen Xu
  # Update time:  13 Aug 2019
  # Usage: 		python score.py --solution_dir=<solution_dir> --prediction_dir=<prediction_dir> --score_dir=<score_dir>
  #           solution_dir contains  e.g. adult.solution
  #           prediction_dir should contain e.g. start.txt, adult.predict_0, adult.predict_1,..., end.txt.
  #           score_dir should contain scores.txt, detailed_results.html
  
  VERSION = 'v20191219'
  DESCRIPTION =\
  """This is the scoring program for AutoDL challenge. It takes the predictions
  made by ingestion program as input and compare to the solution file and produce
  a learning curve.
  Previous updates:
  20191219: [ZY] Write scores.txt file even when failing
  20191204: [ZY] Set scoring waiting time to 30min (1800s)
  20190908: [ZY] Add algebraic operations of learning curves
  20190823: [ZY] Fix the ALC in learning curve: use auc_step instead of auc
  20190820: [ZY] Minor fix: change wait time (for ingestion) from 30s to 90s
  20190709: [ZY] Resolve all issues; rearrange some logging messages;
                 simplify main function; fix exit_code of run_local_test.py;
  20190708: [ZY] Write the class Evaluator for object-oriented scoring program
  20190519: [ZY] Use the correct function for computing AUC of step functions
  20190516: [ZY] Change time budget to 20 minutes.
  20190508: [ZY] Decompose drawing learning curve functions;
                 Remove redirect output feature;
                 Use AUC instead of BAC;
                 Ignore columns with only one class when computing AUC;
                 Use step function instead of trapezoidal rule;
                 Change t0=300 from t0=1 in time transformation:
                   log(1 + t/t0) / log(1 + T/t0)
                 Add second x-axis for the real time in seconds
  20190505: [ZY] Use argparse to parse directories AND time budget;
                 Fix num_preds not updated error.
  20190504: [ZY] Don't raise Exception anymore (for most cases) in order to
                 always have 'Finished' for each submission;
                 Kill ingestion when time limit is exceeded;
                 Use the last modified time of the file 'start.txt' written by
                 ingestion as the start time (`ingestion_start`);
                 Use module-specific logger instead of logging (with root logger);
                 Use the function update_score_and_learning_curve;
  20190429: [ZY] Remove useless code block such as the function is_started;
                 Better code layout.
  20190426.4: [ZY] Fix yaml format in scores.txt (add missing spaces)
  20190426.3: [ZY] Use f.write instead of yaml.dump to write scores.txt
  20190426.2: [ZY] Add logging info when writing scores and learning curves.
  20190426: [ZY] Now write to scores.txt whenever a new prediction is made. This
                 way, participants can still get a score when they exceed time
                 limit (but the submission's status will be marked as 'Failed').
  20190425: [ZY] Add ScoringError and IngestionError: throw error in these cases.
                 Participants will get 'Failed' for their error. But a score will
                 still by computed if applicable.
                 Improve importing order.
                 Log CPU usage.
  20190424: [ZY] Use logging instead of logger; remove start.txt checking.
  20190424: [ZY] Add version and description.
  20190419: [ZY] Judge if ingestion is alive by duration.txt; use logger."""
  
  # Scoring program for the AutoDL challenge
  # Isabelle Guyon and Zhengying Liu, ChaLearn, April 2018-
  
  # ALL INFORMATION, SOFTWARE, DOCUMENTATION, AND DATA ARE PROVIDED "AS-IS".
  # ISABELLE GUYON, CHALEARN, AND/OR OTHER ORGANIZERS OR CODE AUTHORS DISCLAIM
  # ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ANY PARTICULAR PURPOSE, AND THE
  # WARRANTY OF NON-INFRINGEMENT OF ANY THIRD PARTY'S INTELLECTUAL PROPERTY RIGHTS.
  # IN NO EVENT SHALL ISABELLE GUYON AND/OR OTHER ORGANIZERS BE LIABLE FOR ANY SPECIAL,
  # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING OUT OF OR IN
  # CONNECTION WITH THE USE OR PERFORMANCE OF SOFTWARE, DOCUMENTS, MATERIALS,
  # PUBLICATIONS, OR INFORMATION MADE AVAILABLE FOR THE CHALLENGE.
  ################################################################################
  
  
  ################################################################################
  # User defined constants
  ################################################################################
  
  # Verbosity level of logging.
  # Can be: NOTSET, DEBUG, INFO, WARNING, ERROR, CRITICAL
  verbosity_level = 'INFO'
  
  from libscores import read_array, sp, ls, mvmean, tiedrank, _HERE, get_logger
  from os.path import join
  from sys import argv
  from sklearn.metrics import auc, roc_auc_score
  import argparse
  import base64
  import datetime
  import matplotlib; matplotlib.use('Agg') # Solve the Tkinter display issue
  import matplotlib.pyplot as plt
  import numpy as np
  import os
  import psutil
  import sys
  import time
  import yaml
  from random import randrange
  
  logger = get_logger(verbosity_level)
  
  ################################################################################
  # Functions
  ################################################################################
  
  # Metric used to compute the score of a point on the learning curve
  def _autodl_auc(solution, prediction, valid_columns_only=True):
    """Compute normarlized Area under ROC curve (AUC).
    Return Gini index = 2*AUC-1 for  binary classification problems.
    Should work for a vector of binary 0/1 (or -1/1)"solution" and any discriminant values
    for the predictions. If solution and prediction are not vectors, the AUC
    of the columns of the matrices are computed and averaged (with no weight).
    The same for all classification problems (in fact it treats well only the
    binary and multilabel classification problems). When `valid_columns` is not
    `None`, only use a subset of columns for computing the score.
    """
    if valid_columns_only:
      valid_columns = get_valid_columns(solution)
      if len(valid_columns) < solution.shape[-1]:
        logger.info("Some columns in solution have only one class, " +
                       "ignoring these columns for evaluation.")
      solution = solution[:, valid_columns].copy()
      prediction = prediction[:, valid_columns].copy()
    label_num = solution.shape[1]
    auc = np.empty(label_num)
    for k in range(label_num):
      r_ = tiedrank(prediction[:, k])
      s_ = solution[:, k]
      if sum(s_) == 0: print("WARNING: no positive class example in class {}"\
                             .format(k + 1))
      npos = sum(s_ == 1)
      nneg = sum(s_ < 1)
      auc[k] = (sum(r_[s_ == 1]) - npos * (npos + 1) / 2) / (nneg * npos)
    return 2 * mvmean(auc) - 1
  
  
  def autodl_auc(solution, prediction, valid_columns_only=True):
      """More efficient implementation of `_autodl_auc`."""
      if valid_columns_only:
          valid_columns = get_valid_columns(solution)
          if len(valid_columns) < solution.shape[-1]:
              logger.info("Some columns in solution have only one class, " +
                              "ignoring these columns for evaluation.")
              solution = solution[:, valid_columns].copy()
              prediction = prediction[:, valid_columns].copy()
      auc = roc_auc_score(solution, prediction)
      return 2 * auc - 1
  
  
  def accuracy(solution, prediction):
    """Get accuracy of 'prediction' w.r.t true labels 'solution'."""
    epsilon = 1e-15
    # normalize prediction
    prediction_normalized =\
      prediction / (np.sum(np.abs(prediction), axis=1, keepdims=True) + epsilon)
    return np.sum(solution * prediction_normalized) / solution.shape[0]
  
  scoring_functions = {'nauc': autodl_auc,
                       'accuracy': accuracy
                       }
  
  def get_valid_columns(solution):
    """Get a list of column indices for which the column has more than one class.
    This is necessary when computing BAC or AUC which involves true positive and
    true negative in the denominator. When some class is missing, these scores
    don't make sense (or you have to add an epsilon to remedy the situation).
  
    Args:
      solution: array, a matrix of binary entries, of shape
        (num_examples, num_features)
    Returns:
      valid_columns: a list of indices for which the column has more than one
        class.
    """
    num_examples = solution.shape[0]
    col_sum = np.sum(solution, axis=0)
    valid_columns = np.where(1 - np.isclose(col_sum, 0) -
                                 np.isclose(col_sum, num_examples))[0]
    return valid_columns
  
  def is_one_hot_vector(x, axis=None, keepdims=False):
    """Check if a vector 'x' is one-hot (i.e. one entry is 1 and others 0)."""
    norm_1 = np.linalg.norm(x, ord=1, axis=axis, keepdims=keepdims)
    norm_inf = np.linalg.norm(x, ord=np.inf, axis=axis, keepdims=keepdims)
    return np.logical_and(norm_1 == 1, norm_inf == 1)
  
  def is_multiclass(solution):
    """Return if a task is a multi-class classification task, i.e.  each example
    only has one label and thus each binary vector in `solution` only has
    one '1' and all the rest components are '0'.
  
    This function is useful when we want to compute metrics (e.g. accuracy) that
    are only applicable for multi-class task (and not for multi-label task).
  
    Args:
      solution: a numpy.ndarray object of shape [num_examples, num_classes].
    """
    return all(is_one_hot_vector(solution, axis=1))
  
  def get_fig_name(task_name):
    """Helper function for getting learning curve figure name."""
    fig_name = "learning-curve-" + task_name + ".png"
    return fig_name
  
  def get_solution(solution_dir):
    """Get the solution array from solution directory."""
    solution_names = sorted(ls(os.path.join(solution_dir, '*.solution')))
    if len(solution_names) != 1: # Assert only one file is found
      logger.warning("{} solution files found: {}! "\
                     .format(len(solution_names), solution_names) +
                     "Return `None` as solution.")
      return None
    solution_file = solution_names[0]
    solution = read_array(solution_file)
    return solution
  
  def get_task_name(solution_dir):
    """Get the task name from solution directory."""
    solution_names = sorted(ls(os.path.join(solution_dir, '*.solution')))
    if len(solution_names) != 1: # Assert only one file is found
      logger.warning("{} solution files found: {}! "\
                     .format(len(solution_names), solution_names) +
                     "Return `None` as task name.")
      return None
    solution_file = solution_names[0]
    task_name = solution_file.split(os.sep)[-1].split('.')[0]
    return task_name
  
  def transform_time(t, T, t0=None):
    if t0 is None:
      t0 = T
    return np.log(1 + t / t0) / np.log(1 + T / t0)
  
  def auc_step(X, Y):
    """Compute area under curve using step function (in 'post' mode)."""
    if len(X) != len(Y):
      raise ValueError("The length of X and Y should be equal but got " +
                       "{} and {} !".format(len(X), len(Y)))
    area = 0
    for i in range(len(X) - 1):
      delta_X = X[i + 1] - X[i]
      area += delta_X * Y[i]
    return area
  
  def plot_learning_curve(timestamps, scores,
                          start_time=0, time_budget=7200, method='step',
                          transform=None, task_name=None,
                          area_color='cyan', fill_area=True, model_name=None,
                          clear_figure=True, fig=None, show_final_score=True,
                          show_title=True,
                          **kwargs):
    """Plot learning curve using scores and corresponding timestamps.
  
    Args:
      timestamps: iterable of float, each element is the timestamp of
        corresponding performance. These timestamps should be INCREASING.
      scores: iterable of float, scores at each timestamp
      start_time: float, the start time, should be smaller than any timestamp
      time_budget: float, the time budget, should be larger than any timestamp
      method: string, can be one of ['step', 'trapez']
      transform: callable that transform [0, time_budget] into [0, 1]. If `None`,
        use the default transformation
            lambda t: np.log2(1 + t / time_budget)
      task_name: string, name of the task
      area_color: matplotlib color, color of the area under learning curve
      fill_area: boolean, fill the area under the curve or not
      model_name: string, name of the model (learning algorithm).
      clear_figure: boolean, clear previous figures or not
      fig: the figure to plot on
      show_final_score: boolean, show the last score or not
      show_title: boolean, show the plot title or not
      kwargs: Line2D properties, optional
    Returns:
      alc: float, the area under learning curve.
      fig: the figure with learning curve
    Raises:
      ValueError: if the length of `timestamps` and `scores` are not equal,
        or if `timestamps` is not increasing, or if certain timestamp is not in
        the interval [start_time, start_time + time_budget], or if `method` has
        bad values.
    """
    le = len(timestamps)
    if not le == len(scores):
      raise ValueError("The number of timestamps {} ".format(le) +
                       "should be equal to the number of " +
                       "scores {}!".format(len(scores)))
    for i in range(le):
      if i < le - 1 and not timestamps[i] <= timestamps[i + 1]:
        raise ValueError("The timestamps should be increasing! But got " +
                         "[{}, {}] ".format(timestamps[i], timestamps[i + 1]) +
                         "at index [{}, {}].".format(i, i + 1))
      if timestamps[i] < start_time:
        raise ValueError("The timestamp {} at index {}".format(timestamps[i], i) +
                         " is earlier than start time {}!".format(start_time))
    timestamps = [t for t in timestamps if t <= time_budget + start_time]
    if len(timestamps) < le:
      logger.warning("Some predictions are made after the time budget! " +
                     "Ignoring all predictions from the index {}."\
                     .format(len(timestamps)))
      scores = scores[:len(timestamps)]
    if transform is None:
      t0 = 60
      # default transformation
      transform = lambda t: transform_time(t, time_budget, t0=t0)
      xlabel = "Transformed time: " +\
               r'$\tilde{t} = \frac{\log (1 + t / t_0)}{ \log (1 + T / t_0)}$ ' +\
               ' ($T = ' + str(int(time_budget)) + '$, ' +\
               ' $t_0 = ' + str(int(t0)) + '$)'
    else:
      xlabel = "Transformed time: " + r'$\tilde{t}$'
    relative_timestamps = [t - start_time for t in timestamps]
    # Transform X
    X = [transform(t) for t in relative_timestamps]
    Y = list(scores.copy())
    # Add origin as the first point of the curve
    X.insert(0, 0)
    Y.insert(0, 0)
    # Draw learning curve
    if clear_figure:
      plt.clf()
    if fig is None or len(fig.axes) == 0:
      fig = plt.figure(figsize=(7, 7.07))
      ax = fig.add_subplot(111)
      if show_title:
        plt.title("Learning curve for task: {}".format(task_name), y=1.06)
      ax.set_xlabel(xlabel)
      ax.set_xlim(left=0, right=1)
      ax.set_ylabel('score (2 * AUC - 1)')
      ax.set_ylim(bottom=-0.01, top=1)
      ax.grid(True, zorder=5)
      # Show real time in seconds in a second x-axis
      ax2 = ax.twiny()
      ticks = [10, 60, 300, 600, 1200] +\
              list(range(1800, int(time_budget) + 1, 1800))
      ax2.set_xticks([transform(t) for t in ticks])
      ax2.set_xticklabels(ticks)
    ax = fig.axes[0]
    if method == 'step':
      drawstyle = 'steps-post'
      step = 'post'
      auc_func = auc_step
    elif method == 'trapez':
      drawstyle = 'default'
      step = None
      auc_func = auc
    else:
      raise ValueError("The `method` variable should be one of " +
                       "['step', 'trapez']!")
    # Add a point on the final line using last prediction
    X.append(1)
    Y.append(Y[-1])
    # Compute AUC using step function rule or trapezoidal rule
    alc = auc_func(X, Y)
    if model_name:
      label = "{}: ALC={:.4f}".format(model_name, alc)
    else:
      label = "ALC={:.4f}".format(alc)
    # Plot the major part of the figure: the curve
    if 'marker' not in kwargs:
      kwargs['marker'] = 'o'
    if 'markersize' not in kwargs:
      kwargs['markersize'] = 3
    if 'label' not in kwargs:
      kwargs['label'] = label
    ax.plot(X[:-1], Y[:-1], drawstyle=drawstyle, **kwargs)
    # Fill area under the curve
    if fill_area:
      ax.fill_between(X, Y, color='cyan', step=step)
    # Show the latest/final score
    if show_final_score:
      ax.text(X[-1], Y[-1], "{:.4f}".format(Y[-1]))
    # Draw a dotted line from last prediction
    kwargs['linestyle'] = '--'
    kwargs['linewidth'] = 1
    kwargs['marker'] = None
    kwargs.pop('label', None)
    ax.plot(X[-2:], Y[-2:], **kwargs)
    ax.legend()
    return alc, fig
  
  def get_ingestion_info(prediction_dir):
    """Get info on ingestion program: PID, start time, etc. from 'start.txt'.
  
    Args:
      prediction_dir: a string, directory containing predictions (output of
        ingestion)
    Returns:
      A dictionary with keys 'ingestion_pid' and 'start_time' if the file
        'start.txt' exists. Otherwise return `None`.
    """
    start_filepath = os.path.join(prediction_dir, 'start.txt')
    if os.path.exists(start_filepath):
      with open(start_filepath, 'r') as f:
        ingestion_info = yaml.safe_load(f)
      return ingestion_info
    else:
      return None
  
  def get_timestamps(prediction_dir):
    """Read predictions' timestamps stored in 'start.txt'.
  
    The 'start.txt' file should be similar to
      ingestion_pid: 31315
      start_time: 1557269921.7939095
      0: 1557269953.5586617
      1: 1557269956.012751
      2: 1557269958.3
    We see there are 3 predictions. Then this function will return
      start_time, timestamps =
        1557269921.7939095, [1557269953.5586617, 1557269956.012751, 1557269958.3]
    """
    start_filepath = os.path.join(prediction_dir, 'start.txt')
    if os.path.exists(start_filepath):
      with open(start_filepath, 'r') as f:
        ingestion_info = yaml.safe_load(f)
      start_time = ingestion_info['start_time']
      timestamps = []
      idx = 0
      while idx in ingestion_info:
        timestamps.append(ingestion_info[idx])
        idx += 1
      return start_time, timestamps
    else:
      logger.warning("No 'start.txt' file found in the prediction directory " +
                     "{}. Return `None` as timestamps.")
      return None
  
  def get_scores(scoring_function, solution, predictions):
    """Compute a list of scores for a list of predictions.
  
    Args:
      scoring_function: callable with signature
        scoring_function(solution, predictions)
      solution: Numpy array, the solution (true labels).
      predictions: list of array, predictions.
    Returns:
      a list of float, scores
    """
    scores = [scoring_function(solution, pred) for pred in predictions]
    return scores
  
  def compute_scores_bootstrap(scoring_function, solution, prediction, n=10):
      """Compute a list of scores using bootstrap.
  
         Args:
           scoring function: scoring metric taking y_true and y_pred
           solution: ground truth vector
           prediction: proposed solution
           n: number of scores to compute
      """
      scores = []
      l = len(solution)
      for _ in range(n): # number of scoring
        size = solution.shape[0]
        idx = np.random.randint(0, size, size) # bootstrap index
        scores.append(scoring_function(solution[idx], prediction[idx]))
      return scores
  
  def end_file_generated(prediction_dir):
    """Check if ingestion is still alive by checking if the file 'end.txt'
    is generated in the folder of predictions.
    """
    end_filepath =  os.path.join(prediction_dir, 'end.txt')
    logger.debug("CPU usage: {}%".format(psutil.cpu_percent()))
    logger.debug("Virtual memory: {}".format(psutil.virtual_memory()))
    return os.path.isfile(end_filepath)
  
  def is_process_alive(pid):
    """Check if a process is alive according to its PID."""
    try:
      os.kill(pid, 0)
    except OSError:
      return False
    else:
      return True
  
  def terminate_process(pid):
    """Kill a process according to its PID."""
    process = psutil.Process(pid)
    process.terminate()
    logger.debug("Terminated process with pid={} in scoring.".format(pid))
  
  class IngestionError(Exception):
    pass
  
  class ScoringError(Exception):
    pass
  
  class LearningCurve(object):
    """Learning curve object for AutoDL challenges. Contains at least an
    increasing list of float as timestamps and another list of the same length
    of the corresponding score at each timestamp.
    """
  
    def __init__(self, timestamps=None, scores=None, time_budget=1200,
                 score_name=None, task_name=None,
                 participant_name=None, algorithm_name=None, subset='test'):
      """
      Args:
        timestamps: list of float, should be increasing
        scores: list of float, should have the same length as `timestamps`
        time_budget: float, the time budget (for ingestion) of the task
        score_name: string, can be 'nauc' or 'accuracy' (if is multiclass task)
        task_name: string, name of the task, optional
        participant_name: string, name of the participant, optional
        algorithm_name: string, name of the algorithm, optional
      """
      self.timestamps = timestamps or [] # relative timestamps
      self.scores = scores or []
      if len(self.timestamps) != len(self.scores):
        raise ValueError("The number of timestamps should be equal to " +
                         "the number of scores, but got " +
                         "{} and {}".format(len(self.timestamps),
                                            len(self.scores)))
      self.time_budget = time_budget
      self.score_name = score_name or 'nauc'
      self.task_name = task_name
      self.participant_name = participant_name
      self.algorithm_name = algorithm_name
  
    def __repr__(self):
      return "Learning curve for: participant={}, task={}"\
             .format(self.participant_name, self.task_name)
  
    def __add__(self, lc):
      if not isinstance(lc, LearningCurve):
        raise ValueError("Can only add two learning curves but got {}."\
                         .format(type(lc)))
      if self.time_budget != lc.time_budget:
        raise ValueError("Cannot add two learning curves of different " +
                         "time budget: {} and {}!"\
                         .format(self.time_budget, lc.time_budget))
      else:
        time_budget = self.time_budget
      if self.score_name != lc.score_name:
        raise ValueError("Cannot add two learning curves of different " +
                         "score names: {} and {}!"\
                         .format(self.score_name, lc.score_name))
      else:
        score_name = self.score_name
      task_name = self.task_name if self.task_name == lc.task_name else None
      participant_name = self.participant_name \
                         if self.participant_name == lc.participant_name else None
      algorithm_name = self.algorithm_name \
                         if self.algorithm_name == lc.algorithm_name else None
      # Begin merging scores and timestamps
      new_timestamps = []
      new_scores = []
      # Indices of next point to add
      i = 0
      j = 0
      while i < len(self.timestamps) or j < len(lc.timestamps):
        # When two timestamps are close, append only one timestamp
        if i < len(self.timestamps) and j < len(lc.timestamps) and \
          np.isclose(self.timestamps[i], lc.timestamps[j]):
          new_timestamps.append(self.timestamps[i])
          new_scores.append(self.scores[i] + lc.scores[j])
          i += 1
          j += 1
          continue
        # In some cases, use the timestamp/score of this learning curve
        if j == len(lc.timestamps) or \
          (i < len(self.timestamps) and self.timestamps[i] < lc.timestamps[j]):
          new_timestamps.append(self.timestamps[i])
          other_score = 0 if j == 0 else lc.scores[j - 1]
          new_scores.append(self.scores[i] + other_score)
          i += 1
        # In other cases, use the timestamp/score of the other learning curve
        else:
          new_timestamps.append(lc.timestamps[j])
          this_score = 0 if i == 0 else self.scores[i - 1]
          new_scores.append(this_score + lc.scores[j])
          j += 1
      new_lc = LearningCurve(timestamps=new_timestamps,
                             scores=new_scores,
                             time_budget=time_budget,
                             score_name=score_name,
                             task_name=task_name,
                             participant_name=participant_name,
                             algorithm_name=algorithm_name)
      return new_lc
  
    def __mul__(self, real_number):
      if isinstance(real_number, int):
        real_number = float(real_number)
      if not isinstance(real_number, float):
        raise ValueError("Can only multiply a learning curve by a float but got" +
                         " {}.".format(type(real_number)))
      new_scores = [real_number * s for s in self.scores]
      new_lc = LearningCurve(timestamps=self.timestamps,
                             scores=new_scores,
                             time_budget=self.time_budget,
                             score_name=self.score_name,
                             task_name=self.task_name,
                             participant_name=self.participant_name,
                             algorithm_name=self.algorithm_name)
      return new_lc
  
    def __neg__(self):
      return self * (-1)
  
    def __sub__(self, other):
      return self + (-other)
  
    def __truediv__(self, real_number):
      return self * (1 / real_number)
  
    def plot(self, method='step', transform=None,
             area_color='cyan', fill_area=True, model_name=None,
             fig=None, show_final_score=True, **kwargs):
      """Plot the learning curve using `matplotlib.pyplot`.
  
      method: string, can be 'step' or 'trapez'. Decides which drawstyle to use.
          Also effects ALC (Area under Learning Curve)
      transform: callable, for transforming time axis to [0,1] interval, mostly
          optional
      area_color: string or color code, decides the color of the area under curve,
          optional
      fill_area: boolean, whether fill the area under curve with color or not
      model_name: string, if not `None`, will be shown on the legend
      fig: matplotlib.figure.Figure, the figure to plot on. If `None` create a new
          one
      show_final_score: boolean, whether show final score on the figure. Useful
          when overlapping curves
      kwargs: Line2D properties, will be passed for plotting the curve
          see https://matplotlib.org/api/_as_gen/matplotlib.lines.Line2D.html#matplotlib.lines.Line2D
      """
      timestamps = self.timestamps
      scores = self.scores
      time_budget = self.time_budget
      task_name = self.task_name
      alc, fig = plot_learning_curve(timestamps, scores,
                    start_time=0, time_budget=time_budget, method=method,
                    transform=transform, task_name=task_name,
                    area_color=area_color,
                    fill_area=fill_area, model_name=model_name,
                    clear_figure=False, fig=fig,
                    show_final_score=show_final_score, **kwargs)
      return alc, fig
  
    def get_alc(self, t0=60, method='step'):
      X = [transform_time(t, T=self.time_budget, t0=t0)
           for t in self.timestamps]
      Y = list(self.scores.copy())
      X.insert(0, 0)
      Y.insert(0, 0)
      X.append(1)
      Y.append(Y[-1])
      if method == 'step':
        auc_func = auc_step
      elif method == 'trapez':
        auc_func = auc
      alc = auc_func(X, Y)
      return alc
  
    def get_time_used(self):
      if len(self.timestamps) > 0:
        return self.timestamps[-1]
      else:
        return 0
  
    def get_final_score(self):
      if len(self.scores) > 0:
        return self.scores[-1]
      else:
        return 0
  
    def save_figure(self, output_dir):
      alc, ax = self.plot()
      fig_name = get_fig_name(self.task_name)
      path_to_fig = os.path.join(output_dir, fig_name)
      plt.savefig(path_to_fig)
      plt.close()
  
  class Evaluator(object):
  
    def __init__(self, solution_dir=None, prediction_dir=None, score_dir=None,
                 scoring_functions=None, task_name=None, participant_name=None,
                 algorithm_name=None, submission_id=None):
      """
      Args:
        scoring_functions: a dict containing (string, scoring_function) pairs
      """
      self.start_time = time.time()
  
      self.solution_dir = solution_dir
      self.prediction_dir = prediction_dir
      self.score_dir = score_dir
      self.scoring_functions = scoring_functions
  
      self.task_name = task_name or get_task_name(solution_dir)
      self.participant_name = participant_name
      self.algorithm_name = algorithm_name
      self.submission_id = submission_id
  
      # State variables
      self.scoring_success = None
      self.time_limit_exceeded = None
      self.prediction_files_so_far = []
      self.new_prediction_files = []
      self.scores_so_far = {'nauc':[]}
      self.relative_timestamps = []
  
      # Resolve info from directories
      self.solution = self.get_solution()
      # Check if the task is multilabel (i.e. with one hot label)
      self.is_multiclass_task = is_multiclass(self.solution)
  
      self.initialize_learning_curve_page()
      self.fetch_ingestion_info()
      self.learning_curve = self.get_learning_curve()
  
    def get_solution(self):
      """Get solution as NumPy array from `self.solution_dir`."""
      solution = get_solution(self.solution_dir)
      logger.debug("Successfully loaded solution from solution_dir={}"\
                   .format(self.solution_dir))
      return solution
  
    def initialize_learning_curve_page(self):
      """Initialize learning curve page with a message for waiting."""
      # Create the output directory, if it does not already exist
      if not os.path.isdir(self.score_dir):
        os.mkdir(self.score_dir)
      # Initialize detailed_results.html (learning curve page)
      detailed_results_filepath = os.path.join(self.score_dir,
                                               'detailed_results.html')
      html_head = '<html><head> <meta http-equiv="refresh" content="5"> ' +\
                  '</head><body><pre>'
      html_end = '</pre></body></html>'
      with open(detailed_results_filepath, 'a') as html_file:
        html_file.write(html_head)
        html_file.write("Starting training process... <br> Please be patient. " +
                        "Learning curves will be generated when first " +
                        "predictions are made.")
        html_file.write(html_end)
  
    def fetch_ingestion_info(self):
      """Resolve some information from output of ingestion program. This includes
      especially: `ingestion_start`, `ingestion_pid`, `time_budget`.
  
      Raises:
        IngestionError if no sign of ingestion starting detected after 1800
        seconds.
      """
      logger.debug("Fetching ingestion info...")
      prediction_dir = self.prediction_dir
      # Wait 1800 seconds for ingestion to start and write 'start.txt',
      # Otherwise, raise an exception.
      wait_time = 1800
      ingestion_info = None
      for i in range(wait_time):
        ingestion_info = get_ingestion_info(prediction_dir)
        if not ingestion_info is None:
          logger.info("Detected the start of ingestion after {} ".format(i) +
                      "seconds. Start scoring.")
          break
        time.sleep(1)
      else:
        raise IngestionError("[-] Failed: scoring didn't detected the start of " +
                             "ingestion after {} seconds.".format(wait_time))
      # Get ingestion start time
      ingestion_start = ingestion_info['start_time']
      # Get ingestion PID
      ingestion_pid = ingestion_info['ingestion_pid']
      # Get time_budget for ingestion
      assert 'time_budget' in ingestion_info
      time_budget = ingestion_info['time_budget']
      # Set attributes
      self.ingestion_info = ingestion_info
      self.ingestion_start = ingestion_start
      self.ingestion_pid = ingestion_pid
      self.time_budget = time_budget
      logger.debug("Ingestion start time: {}".format(ingestion_start))
      logger.debug("Scoring start time: {}".format(self.start_time))
      logger.debug("Ingestion info successfully fetched.")
  
    def end_file_generated(self):
      return end_file_generated(self.prediction_dir)
  
    def ingestion_is_alive(self):
      return is_process_alive(self.ingestion_pid)
  
    def kill_ingestion(self):
      terminate_process(self.ingestion_pid)
      assert not self.ingestion_is_alive()
  
    def prediction_filename_pattern(self):
      return "{}.predict_*".format(self.task_name)
  
    def get_new_prediction_files(self):
      """Fetch new prediction file(name)s found in prediction directory and update
      corresponding attributes.
  
      Examples of prediction file name: mini.predict_0, mini.predict_1
  
      Returns:
        List of new prediction files found.
      """
      prediction_files = ls(os.path.join(self.prediction_dir,
                                         self.prediction_filename_pattern()))
      logger.debug("Prediction files: {}".format(prediction_files))
      new_prediction_files = [p for p in prediction_files
                              if p not in self.prediction_files_so_far]
      order_key = lambda filename: int(filename.split('_')[-1])
      self.new_prediction_files = sorted(new_prediction_files, key=order_key)
      return self.new_prediction_files
  
    def compute_score_per_prediction(self):
      """For new predictions found, compute their score using `self.solution`
      and scoring functions in `self.scoring_functions`. Then concatenate
      the list of new predictions to the list of resolved predictions so far.
      """
      for score_name in self.scoring_functions:
        scoring_function = self.scoring_functions[score_name]
        if score_name != 'accuracy' or self.is_multiclass_task:
          new_scores = [scoring_function(self.solution, read_array(pred))
                        for pred in self.new_prediction_files]
          if score_name in self.scores_so_far:
            self.scores_so_far[score_name] += new_scores
          else:
            self.scores_so_far[score_name] = new_scores
      # If new predictions are found, update state variables
      if self.new_prediction_files:
        self.prediction_files_so_far += self.new_prediction_files
        num_preds = len(self.prediction_files_so_far)
        self.relative_timestamps = self.get_relative_timestamps()[:num_preds]
        self.learning_curve = self.get_learning_curve()
        self.new_prediction_files = []
  
    def get_relative_timestamps(self):
      """Get a list of relative timestamps. The beginning has relative timestamp
      zero.
      """
      ingestion_start, timestamps = get_timestamps(self.prediction_dir)
      relative_timestamps = [t - ingestion_start for t in timestamps]
      return relative_timestamps
  
    def write_score(self):
      """Write score and duration to score_dir/scores.txt"""
      score_dir = self.score_dir
      score = self.learning_curve.get_alc()
      duration = self.learning_curve.get_time_used()
      score_filename = os.path.join(score_dir, 'scores.txt')
      score_info_dict = {'score': score, # ALC
                         'Duration': duration,
                         'task_name': self.task_name,
                         'timestamps': self.relative_timestamps,
                         'nauc_scores': self.scores_so_far['nauc']
                        }
      if 'accuracy' in self.scores_so_far:
        score_info_dict['accuracy'] = self.scores_so_far['accuracy']
      with open(score_filename, 'w') as f:
        f.write('score: ' + str(score) + '\n')
        f.write('Duration: ' + str(duration) + '\n')
        f.write('timestamps: {}\n'.format(self.relative_timestamps))
        f.write('nauc_scores: {}\n'.format(self.scores_so_far['nauc']))
        if 'accuracy' in self.scores_so_far:
          f.write('accuracy: {}\n'.format(self.scores_so_far['accuracy']))
      logger.debug("Wrote to score_filename={} with score={}, duration={}"\
                    .format(score_filename, score, duration))
      return score_info_dict
  
    def append_score(self, scores_dict):
        """Add fields and values to scores.txt"""
        score_filename = os.path.join(self.score_dir, 'scores.txt')
        with open(score_filename, 'a') as f:
            for key, value in scores_dict.items():
                f.write('{}: '.format(key) + str(value) + '\n')
        return scores_dict
  
    def write_scores_html(self, auto_refresh=True, append=False):
      score_dir = self.score_dir
      filename = 'detailed_results.html'
      image_paths = sorted(ls(os.path.join(score_dir, '*.png')))
      if auto_refresh:
        html_head = '<html><head> <meta http-equiv="refresh" content="5"> ' +\
                    '</head><body><pre>'
      else:
        html_head = """<html><body><pre>"""
      html_end = '</pre></body></html>'
      if append:
        mode = 'a'
      else:
        mode = 'w'
      filepath = os.path.join(score_dir, filename)
      with open(filepath, mode) as html_file:
          html_file.write(html_head)
          for image_path in image_paths:
            with open(image_path, "rb") as image_file:
              encoded_string = base64.b64encode(image_file.read())
              encoded_string = encoded_string.decode('utf-8')
              s = '<img src="data:image/png;charset=utf-8;base64,{}"/>'\
                  .format(encoded_string)
              html_file.write(s + '<br>')
          html_file.write(html_end)
      logger.debug("Wrote learning curve page to {}".format(filepath))
  
    def get_learning_curve(self, score_name='nauc'):
      timestamps = self.relative_timestamps
      scores = self.scores_so_far[score_name]
      return LearningCurve(timestamps=timestamps, scores=scores,
                           time_budget=self.time_budget,
                           score_name=score_name, task_name=self.task_name,
                           participant_name=self.participant_name,
                           algorithm_name=self.algorithm_name)
  
    def draw_learning_curve(self, **kwargs):
      """Draw learning curve for one task and save to `score_dir`."""
      self.compute_score_per_prediction()
      scores = self.scores_so_far['nauc']
      is_multiclass_task = self.is_multiclass_task
      timestamps = self.get_relative_timestamps()
      sorted_pairs = sorted(zip(timestamps, scores))
      start = 0
      time_used = -1
      if len(timestamps) > 0:
        time_used = sorted_pairs[-1][0] - start
        latest_score = sorted_pairs[-1][1]
        if is_multiclass_task:
          accuracy_scores = self.scores_so_far['accuracy']
          sorted_pairs_acc = sorted(zip(timestamps, accuracy_scores))
          latest_acc = sorted_pairs_acc[-1][1]
      X = [t for t, _ in sorted_pairs]
      Y = [s for _, s in sorted_pairs]
      alc, fig = plot_learning_curve(X, Y, time_budget=self.time_budget,
                              task_name=self.task_name, **kwargs)
      fig_name = get_fig_name(self.task_name)
      path_to_fig = os.path.join(self.score_dir, fig_name)
      plt.savefig(path_to_fig)
      plt.close()
      return alc, time_used
  
    def update_score_and_learning_curve(self):
      self.draw_learning_curve()
      # Update learning curve page (detailed_results.html)
      self.write_scores_html()
      # Write score
      score = self.write_score()['score']
      return score
  
    def compute_error_bars(self, n=10):
      """Compute error bars on evaluation with bootstrap.
  
      Args:
          n: number of times to compute the score (more means more precision)
      Returns:
          (mean, std, var)
      """
      try:
          scoring_function = self.scoring_functions['nauc']
          solution = self.solution
          last_prediction = read_array(self.prediction_files_so_far[-1])
          scores = compute_scores_bootstrap(scoring_function, solution, last_prediction, n=n)
          return np.mean(scores), np.std(scores), np.var(scores)
      except: # not able to compute error bars
          return -1, -1, -1
  
    def compute_alc_error_bars(self, n=10):
        """ Return mean, std and variance of ALC score with n runs.
            n curves are created:
                For each timestamp, the value of AUC is computed from boostraps of y_true and y_pred.
                During one curve building, we keep the same boostrap index for each prediction timestamp.
  
            Args:
                n: number of times to compute the score (more means more precision)
            Returns:
                (mean, std, var)
        """
        try:
            scoring_function = self.scoring_functions['nauc']
            solution = self.solution
            alc_scores = []
            for _ in range(n): # n learning curves to compute
                scores = []
                size = solution.shape[0]
                idx = np.random.randint(0, size, size) # bootstrap index
                for prediction_file in self.prediction_files_so_far:
                    prediction = read_array(prediction_file)
                    scores.append(scoring_function(solution[idx], prediction[idx]))
                # create new learning curve
                learning_curve = LearningCurve(timestamps=self.relative_timestamps, # self.learning_curve.timestamps,
                                               scores=scores, # list of AUC scores
                                               time_budget=self.time_budget)
                alc_scores.append(learning_curve.get_alc())
            return np.mean(alc_scores), np.std(alc_scores), np.var(alc_scores)
        except: # not able to compute error bars
            return -1, -1, -1
  
    def score_new_predictions(self):
      new_prediction_files = evaluator.get_new_prediction_files()
      if len(new_prediction_files) > 0:
        score = evaluator.update_score_and_learning_curve()
        logger.info("[+] New prediction found. Now number of predictions " +
                     "made = {}"\
                     .format(len(evaluator.prediction_files_so_far)))
        logger.info("Current area under learning curve for {}: {:.4f}"\
                  .format(evaluator.task_name, score))
        logger.info("(2 * AUC - 1) of the latest prediction is {:.4f}."\
                  .format(evaluator.scores_so_far['nauc'][-1]))
        if evaluator.is_multiclass_task:
          logger.info("Accuracy of the latest prediction is {:.4f}."\
                    .format(evaluator.scores_so_far['accuracy'][-1]))
  
  # =============================== MAIN ========================================
  
  if __name__ == "__main__":
      logger.info("="*5 + " Start scoring program. " +
                  "Version: {} ".format(VERSION) + "="*5)
  
      # Default I/O directories:
      root_dir = _HERE(os.pardir)
      default_solution_dir = join(root_dir, "AutoDL_sample_data")
      default_prediction_dir = join(root_dir, "AutoDL_sample_result_submission")
      default_score_dir = join(root_dir, "AutoDL_scoring_output")
  
      # Parse directories from input arguments
      parser = argparse.ArgumentParser()
      parser.add_argument('--solution_dir', type=str,
                          default=default_solution_dir,
                          help="Directory storing the solution with true " +
                               "labels, e.g. adult.solution.")
      parser.add_argument('--prediction_dir', type=str,
                          default=default_prediction_dir,
                          help="Directory storing the predictions. It should" +
                               "contain e.g. [start.txt, adult.predict_0, " +
                               "adult.predict_1, ..., end.txt].")
      parser.add_argument('--score_dir', type=str,
                          default=default_score_dir,
                          help="Directory storing the scoring output " +
                               "e.g. `scores.txt` and `detailed_results.html`.")
      args = parser.parse_args()
      logger.debug("Parsed args are: " + str(args))
      logger.debug("-" * 50)
      solution_dir = args.solution_dir
      prediction_dir = args.prediction_dir
      score_dir = args.score_dir
  
      logger.debug("Version: {}. Description: {}".format(VERSION, DESCRIPTION))
      logger.debug("Using solution_dir: " + str(solution_dir))
      logger.debug("Using prediction_dir: " + str(prediction_dir))
      logger.debug("Using score_dir: " + str(score_dir))
  
      #################################################################
      # Initialize an evaluator (scoring program) object
      evaluator = Evaluator(solution_dir, prediction_dir, score_dir,
                            scoring_functions=scoring_functions)
      #################################################################
  
      ingestion_start = evaluator.ingestion_start
      time_budget = evaluator.time_budget
  
      try:
        while(time.time() < ingestion_start + time_budget):
          if evaluator.end_file_generated():
            logger.info("Detected ingestion program had stopped running " +
                        "because an 'end.txt' file is written by ingestion. " +
                        "Stop scoring now.")
            evaluator.scoring_success = True
            break
          time.sleep(1)
  
          ### Fetch new predictions, compute their scores and update variables ###
          evaluator.score_new_predictions()
          ########################################################################
  
          logger.debug("Prediction files so far: {}"\
                       .format(evaluator.prediction_files_so_far))
        else: # When time budget is used up, kill ingestion
          if evaluator.ingestion_is_alive():
            evaluator.time_limit_exceeded = True
            evaluator.kill_ingestion()
            logger.info("Detected time budget is used up. Killed ingestion and " +
                        "terminating scoring...")
      except Exception as e:
        evaluator.scoring_success = False
        logger.error("[-] Error occurred in scoring:\n" + str(e),
                      exc_info=True)
  
      evaluator.score_new_predictions()
  
      logger.info("Final area under learning curve for {}: {:.4f}"\
                .format(evaluator.task_name, evaluator.learning_curve.get_alc()))
  
      # Write one last time the detailed results page without auto-refreshing
      evaluator.write_scores_html(auto_refresh=False)
  
      # Compute scoring error bars of last prediction
      n = 10
      logger.info("Computing error bars with {} scorings...".format(n))
      nauc_mean, nauc_std, nauc_var = evaluator.compute_error_bars(n=n)
      logger.info("\nLatest prediction NAUC:\n* Mean: {}\n* Standard deviation: {}\n* Variance: {}".format(nauc_mean, nauc_std, nauc_var))
  
      # Compute ALC error bars
      n = 5
      logger.info("Computing ALC error bars with {} curves...".format(n))
      alc_mean, alc_std, alc_var = evaluator.compute_alc_error_bars(n=n)
      logger.info("\nArea under Learning Curve:\n* Mean: {}\n* Standard deviation: {}\n* Variance: {}".format(alc_mean, alc_std, alc_var))
  
      # Write error bars into scores.txt file
      scores_dict = {"nauc_mean":nauc_mean, "nauc_std":nauc_std, "nauc_var":nauc_var,
                     "alc_mean":alc_mean, "alc_std":alc_std, "alc_var":alc_var}
      evaluator.append_score(scores_dict)
  
      scoring_start = evaluator.start_time
      # Use 'end.txt' file to detect if ingestion program ends
      end_filepath =  os.path.join(prediction_dir, 'end.txt')
      if not evaluator.scoring_success is None and not evaluator.scoring_success:
        logger.error("[-] Some error occurred in scoring program. " +
                    "Please see output/error log of Scoring Step.")
      elif not os.path.isfile(end_filepath):
        if evaluator.time_limit_exceeded:
          logger.error("[-] Ingestion program exceeded time budget. " +
                       "Predictions made so far will be used for evaluation.")
        else: # Less probable to fall in this case
          if evaluator.ingestion_is_alive():
            evaluator.kill_ingestion()
          logger.error("[-] No 'end.txt' file is produced by ingestion. " +
                       "Ingestion or scoring may have not terminated normally.")
      else:
        with open(end_filepath, 'r') as f:
          end_info_dict = yaml.safe_load(f)
        ingestion_duration = end_info_dict['ingestion_duration']
  
        if end_info_dict['ingestion_success'] == 0:
          evaluator.write_score()
          logger.error("[-] Some error occurred in ingestion program. " +
                      "Please see output/error log of Ingestion Step.")
        else:
          logger.info("[+] Successfully finished scoring! " +
                    "Scoring duration: {:.2f} sec. "\
                    .format(time.time() - scoring_start) +
                    "Ingestion duration: {:.2f} sec. "\
                    .format(ingestion_duration) +
                    "The score of your algorithm on the task '{}' is: {:.6f}."\
                    .format(evaluator.task_name,
                            evaluator.learning_curve.get_alc()))
  
      logger.info("[Scoring terminated]")
  