class_docstr: Ranking metric.
class_docstr_tok: ["Ranking","metric","."]
docstr: Assign a score to the ranking of the references of an instance.
docstr_tok: ["Assign","a","score","to","the","ranking","of","the","references","of","an","instance","."]
code: def evaluate_references(
      self,
      adapter_spec: AdapterSpec,
      reference_request_states: List[RequestState],
      metric_service: MetricService,
      eval_cache_path: str,
  ) -> List[Stat]:
      """Assign a score to the ranking of the references of an instance."""
      # Extract relevant data from the request states.
      ranking_objects: List[RankingObject] = [
          self.create_ranking_object(rs)
          for rs in reference_request_states
      ]
  
      # Populate the model relevance of the ranking objects.
      ranking_objects = self.model_relevance_fn(ranking_objects)
  
      # Get ground truth relevances.
      true_relevances: Dict[str, int] = self.get_true_relevances(
          ranking_objects
      )
  
      # Get relevance values for each reference based on the model output and compute measures.
      run_relevances: Dict[str, int] = self.get_run_relevances(
          ranking_objects
      )
      stats: List[Stat] = self.compute_measures(
          true_relevances, run_relevances
      )
  
      # Get relevance values for each reference with rank up to self.rank based on model output and compute measures.
      if self.rank:
          run_relevances = self.get_run_relevances(
              ranking_objects, rank_limit=self.rank
          )
          mn_suffix = f" (topk={self.rank})"
          stats_with_rank_limit = self.compute_measures(
              true_relevances,
              run_relevances,
              metric_name_suffix=mn_suffix,
          )
          stats += stats_with_rank_limit
  
      # Add reference ranks as stats
      # - Ranking objects with higher model relevance should have a better
      #   ranking, which is why we are setting the ranking of an object to be
      #   len(ranking_objects) minus its relevance.
      stats += [
          Stat(MetricName(f"ref{r.reference_index}_rank")).add(
              len(ranking_objects) - r.model_relevance  # type: ignore
          )
          for r in ranking_objects
      ]
  
      return stats
  
imports: from dataclasses import dataclass
  from typing import Callable, Dict, List, Tuple, Optional
  import pytrec_eval
  from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_RANKING_BINARY
  from helm.benchmark.adaptation.request_state import RequestState
  from helm.benchmark.adaptation.adapter_spec import AdapterSpec
  from helm.benchmark.scenarios.scenario import unpack_tag, CORRECT_TAG, Reference
  from helm.common.request import RequestResult
  from helm.common.general import binarize_dict
  from .metric import Metric
  from .metric_name import MetricName
  from .metric_service import MetricService
  from .statistic import Stat
label_desc: Use image node_id 17 for calling the RankingMetric obj's underlying member method code with example usage: obj.evaluate_references(adapter_spec, reference_request_states, metric_service, eval_cache_path) and returns: stats