class_docstr: Ranking metric.
class_docstr_tok: ["Ranking","metric","."]
docstr: Constructor for the RankingMetric.
  
  Args:
      method: The adaptation method used. The method must exists in
          self.METHOD_LIST.
      measure_names: The trec_eval measure names that will be computed.
          Measure names must be measure names supported by the official
          trec_eval measure. List of supported measures can be found in
          self.SUPPORTED_MEASURES. Note that:
              (1) We also accept the parametrized versions
                  (e.g. "measure_name.k") of self.SUPPORTED_MEASURES
                  measures.
              (2) We accept any measure that's in either "measure_name" or
                  "measure_name.k" form, where measure_name is in
                  pytrec_eval.supported_measures, but note that
                  self.BINARY_MEASURES list must be modified to
                  include any new binary measures.
      correct_output: If the ADAPT_RANKING_BINARY mode is selected,
          the string that should be outputted if the model predicts that
          the object given in the instance can answer the question.
      wrong_output: If the ADAPT_RANKING_BINARY mode is selected, the
          string that should be outputted if the model predicts that the
          object given in the instance can not answer the question.
      rank: The optional number of max top document rankings to keep for
          evaluation. If None, all the rankings are evaluated. If
          specified, only the documents that have a rank up to and
          including the specified rank are evaluated.
      multiple_relevance_values: Query relevance values can either be
          binary or take on multiple values, as explained below. This flag
          indicates whether the relevance values can take multiple values.
              (1) Binary relevance values: If the relevance values are
                  binary, it means that all the matching relationships
                  would get assigned a relevance value of 1, while the
                  known non-matching relationships would get assigned a
                  relevance value of 0.
              (2) Multiple relevance values: In the case of multiple
                  relevance values, the value of 0 will be interpreted as
                  non-matching relationship, but any other value would be
                  interpreted as a matching relationship differing
                  strengths.
docstr_tok: ["Constructor","for","the","RankingMetric",".","Args",":","method",":","The","adaptation","method","used",".","The","method","must","exists","in","self.METHOD_LIST",".","measure_names",":","The","trec_eval","measure","names","that","will","be","computed",".","Measure","names","must","be","measure","names","supported","by","the","official","trec_eval","measure",".","List","of","supported","measures","can","be","found","in","self.SUPPORTED_MEASURES",".","Note","that",":","(","1",")","We","also","accept","the","parametrized","versions","(","e.g",".","``","measure_name.k","''",")","of","self.SUPPORTED_MEASURES","measures",".","(","2",")","We","accept","any","measure","that","'s","in","either","``","measure_name","''","or","``","measure_name.k","''","form",",","where","measure_name","is","in","pytrec_eval.supported_measures",",","but","note","that","self.BINARY_MEASURES","list","must","be","modified","to","include","any","new","binary","measures",".","correct_output",":","If","the","ADAPT_RANKING_BINARY","mode","is","selected",",","the","string","that","should","be","outputted","if","the","model","predicts","that","the","object","given","in","the","instance","can","answer","the","question",".","wrong_output",":","If","the","ADAPT_RANKING_BINARY","mode","is","selected",",","the","string","that","should","be","outputted","if","the","model","predicts","that","the","object","given","in","the","instance","can","not","answer","the","question",".","rank",":","The","optional","number","of","max","top","document","rankings","to","keep","for","evaluation",".","If","None",",","all","the","rankings","are","evaluated",".","If","specified",",","only","the","documents","that","have","a","rank","up","to","and","including","the","specified","rank","are","evaluated",".","multiple_relevance_values",":","Query","relevance","values","can","either","be","binary","or","take","on","multiple","values",",","as","explained","below",".","This","flag","indicates","whether","the","relevance","values","can","take","multiple","values",".","(","1",")","Binary","relevance","values",":","If","the","relevance","values","are","binary",",","it","means","that","all","the","matching","relationships","would","get","assigned","a","relevance","value","of","1",",","while","the","known","non-matching","relationships","would","get","assigned","a","relevance","value","of","0",".","(","2",")","Multiple","relevance","values",":","In","the","case","of","multiple","relevance","values",",","the","value","of","0","will","be","interpreted","as","non-matching","relationship",",","but","any","other","value","would","be","interpreted","as","a","matching","relationship","differing","strengths","."]
code: def __init__(
      self,
      method: str,
      measure_names: List[str],
      correct_output: str,
      wrong_output: str,
      rank: Optional[int] = None,
      multiple_relevance_values: bool = False,
  ):
      """Constructor for the RankingMetric.
  
      Args:
          method: The adaptation method used. The method must exists in
              self.METHOD_LIST.
          measure_names: The trec_eval measure names that will be computed.
              Measure names must be measure names supported by the official
              trec_eval measure. List of supported measures can be found in
              self.SUPPORTED_MEASURES. Note that:
                  (1) We also accept the parametrized versions
                      (e.g. "measure_name.k") of self.SUPPORTED_MEASURES
                      measures.
                  (2) We accept any measure that's in either "measure_name" or
                      "measure_name.k" form, where measure_name is in
                      pytrec_eval.supported_measures, but note that
                      self.BINARY_MEASURES list must be modified to
                      include any new binary measures.
          correct_output: If the ADAPT_RANKING_BINARY mode is selected,
              the string that should be outputted if the model predicts that
              the object given in the instance can answer the question.
          wrong_output: If the ADAPT_RANKING_BINARY mode is selected, the
              string that should be outputted if the model predicts that the
              object given in the instance can not answer the question.
          rank: The optional number of max top document rankings to keep for
              evaluation. If None, all the rankings are evaluated. If
              specified, only the documents that have a rank up to and
              including the specified rank are evaluated.
          multiple_relevance_values: Query relevance values can either be
              binary or take on multiple values, as explained below. This flag
              indicates whether the relevance values can take multiple values.
                  (1) Binary relevance values: If the relevance values are
                      binary, it means that all the matching relationships
                      would get assigned a relevance value of 1, while the
                      known non-matching relationships would get assigned a
                      relevance value of 0.
                  (2) Multiple relevance values: In the case of multiple
                      relevance values, the value of 0 will be interpreted as
                      non-matching relationship, but any other value would be
                      interpreted as a matching relationship differing
                      strengths.
      """
      # Input validation
      assert (
          method in self.METHOD_LIST
      ), f"Mode must be one of {self.METHOD_LIST}, instead found {method}."
      self.method = method
  
      for measure_name in measure_names:
          assert self.validate_measure_name(
              measure_name
          ), f"""Measure name {measure_name} isn't supported."""
      self.measure_names = measure_names
  
      self.correct_output = correct_output
      self.wrong_output = wrong_output
  
      if rank:
          assert rank >= 0
      self.rank = rank
  
      self.multiple_relevance_values = multiple_relevance_values
  
      # Decide which model relevance function to use
      METHOD_TO_RELEVANCE_FUNCTION: Dict[
          str, Callable[[List[RankingObject]], List[RankingObject]]
      ] = {
          ADAPT_RANKING_BINARY: self.compute_model_relevance_binary_ranking
      }
      self.model_relevance_fn = METHOD_TO_RELEVANCE_FUNCTION[
          self.method
      ]
  
imports: from dataclasses import dataclass
  from typing import Callable, Dict, List, Tuple, Optional
  import pytrec_eval
  from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_RANKING_BINARY
  from helm.benchmark.adaptation.request_state import RequestState
  from helm.benchmark.adaptation.adapter_spec import AdapterSpec
  from helm.benchmark.scenarios.scenario import unpack_tag, CORRECT_TAG, Reference
  from helm.common.request import RequestResult
  from helm.common.general import binarize_dict
  from .metric import Metric
  from .metric_name import MetricName
  from .metric_service import MetricService
  from .statistic import Stat
label_desc: Use image node_id 1 to create a new RankingMetric object from inherited base classes: Metric with example: obj = RankingMetric(method, measure_names, correct_output, wrong_output, rank, multiple_relevance_values)