text: import numpy as np
  import math
  
  
  # (C) Mathieu Blondel, November 2013
  # License: BSD 3 clause
  # https://gist.github.com/mblondel/7337391
  def dcg_score(y_true, y_score, k=10, gains="exponential"):
      """Discounted cumulative gain (DCG) at rank k
      Parameters
      ----------
      y_true : array-like, shape = [n_samples]
          Ground truth (true relevance labels).
      y_score : array-like, shape = [n_samples]
          Predicted scores.
      k : int
          Rank.
      gains : str
          Whether gains should be "exponential" (default) or "linear".
      Returns
      -------
      DCG @k : float
      """
      order = np.argsort(y_score)[::-1]
      y_true = np.take(y_true, order[:k])
  
      if gains == "exponential":
          gains = 2 ** y_true - 1
      elif gains == "linear":
          gains = y_true
      else:
          raise ValueError("Invalid gains option.")
  
      # highest rank is 1 so +2 instead of +1
      discounts = np.log2(np.arange(len(y_true)) + 1)
      return np.sum(gains / discounts)
  
  
  def ndcg_score(y_true, y_score, k=10, gains="exponential"):
      """Normalized discounted cumulative gain (NDCG) at rank k
      Parameters
      ----------
      y_true : array-like, shape = [n_samples]
          Ground truth (true relevance labels).
      y_score : array-like, shape = [n_samples]
          Predicted scores.
      k : int
          Rank.
      gains : str
          Whether gains should be "exponential" (default) or "linear".
      Returns
      -------
      NDCG @k : float
      """
      best = dcg_score(y_true, y_true, k, gains)
      actual = dcg_score(y_true, y_score, k, gains)
      return actual / best
  
  
  def find_dcg(element_list):
      """
      Discounted Cumulative Gain (DCG)
      The definition of DCG can be found in this paper:
          Azzah Al-Maskari, Mark Sanderson, and Paul Clough. 2007.
          "The relationship between IR effectiveness measures and user satisfaction."
      Parameters:
          element_list - a list of ranks Ex: [5,4,2,2,1]
      Returns:
          score
      """
      score = 0.0
      for order, rank in enumerate(element_list):
          score += float(rank)/math.log((order+2))
      return score
  
  
  def find_ndcg(reference, hypothesis):
      """
      Normalized Discounted Cumulative Gain (nDCG)
      Normalized version of DCG:
          nDCG = DCG(hypothesis)/DCG(reference)
      Parameters:
          reference   - a gold standard (perfect) ordering Ex: [5,4,3,2,1]
          hypothesis  - a proposed ordering Ex: [5,2,2,3,1]
      Returns:
          ndcg_score  - normalized score
      """
  
      return find_dcg(hypothesis)/find_dcg(reference)
  
  
  def dcg_at_k(r, k, method=0):
      """Score is discounted cumulative gain (dcg)
      Relevance is positive real values.  Can use binary
      as the previous methods.
      There is a typographical error on the formula referenced in the original definition of this function:
      http://www.stanford.edu/class/cs276/handouts/EvaluationNew-handout-6-per.pdf
      log2(i) should be log2(i+1)
  
      The formulas here are derived from
      https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Discounted_Cumulative_Gain
  
      The formulas return the same results when r contains only binary values
  
      Args:
          r: Relevance scores (list or numpy array) in rank order
              (first element is the most relevant item)
          k: Number of results to consider
          method: If 0 then sum rel_i / log2(i + 1) [not log2(i)]
                  If 1 then sum (2^rel_i - 1) / log2(i + 1)
  
      Returns:
          Discounted cumulative gain
      """
      r = np.asfarray(r)[:k]
      if r.size:
          if method == 0:
              return np.sum(r / np.log2(np.arange(2, r.size + 2)))
          elif method == 1:
              return np.sum(np.subtract(np.power(2, r), 1) / np.log2(np.arange(2, r.size + 2)))
          else:
              raise ValueError('method must be 0 or 1.')
      return 0.
  
  
  def ndcg_at_k(r, k, method=0):
      """Score is normalized discounted cumulative gain (ndcg)
      Relevance is positive real values.  Can use binary
      as the previous methods.
      Args:
          r: Relevance scores (list or numpy array) in rank order
              (first element is the most relevant item)
          k: Number of results to consider
          method: If 0 then sum rel_i / log2(i + 1) [not log2(i)]
                  If 1 then sum (2^rel_i - 1) / log2(i + 1)
  
      Returns:
          Normalized discounted cumulative gain
      """
      dcg_max = dcg_at_k(sorted(r, reverse=True), k, method)
      if not dcg_max:
          return 0.
      return dcg_at_k(r, k, method) / dcg_max
  
  
  def precision_at_k(r, k):
      """Score is precision @ k
      Relevance is binary (nonzero is relevant).
      Traceback (most recent call last):
          File "<stdin>", line 1, in ?
      ValueError: Relevance score length < k
      Args:
          r: Relevance scores (list or numpy) in rank order
              (first element is the first item)
      Returns:
          Precision @ k
      Raises:
          ValueError: len(r) must be >= k
      """
      assert k >= 1
      r = np.asarray(r)[:k] != 0
      if r.size != k:
          raise ValueError('Relevance score length < k')
      return np.mean(r)
  
  
  def average_precision(r):
      """Score is average precision (area under PR curve)
      Relevance is binary (nonzero is relevant).
      Args:
          r: Relevance scores (list or numpy) in rank order
              (first element is the first item)
      Returns:
          Average precision
      """
      r = np.asarray(r) != 0
      out = [precision_at_k(r, k + 1) for k in range(r.size) if r[k]]
      if not out:
          return 0.
      return np.mean(out)
  
  
  def mean_average_precision(rs):
      """Score is mean average precision
      Relevance is binary (nonzero is relevant).
      Args:
          rs: Iterator of relevance scores (list or numpy) in rank order
              (first element is the first item)
      Returns:
          Mean average precision
      """
      return np.mean([average_precision(r) for r in rs])
  
  