class_docstr: Reimplementation of SummarizationMetric's evals using critique evaluation.
  
  This is a demonstration of critique evaluation and is not intended for production use.
class_docstr_tok: ["Reimplementation","of","SummarizationMetric","'s","evals","using","critique","evaluation",".","This","is","a","demonstration","of","critique","evaluation","and","is","not","intended","for","production","use","."]
docstr: Get critiques of a summary and compute metrics based on the critiques.
docstr_tok: ["Get","critiques","of","a","summary","and","compute","metrics","based","on","the","critiques","."]
code: def evaluate_generation(
      self,
      adapter_spec: AdapterSpec,
      request_state: RequestState,
      metric_service: MetricService,
      eval_cache_path: str,
  ) -> List[Stat]:
      """Get critiques of a summary and compute metrics based on the critiques."""
      assert request_state.result is not None
      if len(request_state.result.completions) != 1:
          raise ValueError(
              "SummarizationCritiqueMetric only supports a single generation per instance"
          )
      summary = request_state.result.completions[0].text
      request = CritiqueRequest(
          self._template,
          fields={
              "original_text": request_state.instance.input.text,
              "summary": summary,
          },
      )
      result = metric_service.make_critique_request(request)
      if not result or not result.responses:
          return []
      stats: Dict[str, Stat] = {}
      for question in self._template.questions:
          stats[question.name] = Stat(MetricName(question.name))
  
      # Skip computing metrics if there are not enough responses.
      if len(result.responses) < request.template.num_respondents:
          return []
  
      for response in result.responses:
          for answer_name, answer in response.answers.items():
              if not isinstance(answer, str):
                  raise ValueError(
                      f"Expected answer to {answer_name} be a string"
                  )
              answer_value: float
              if answer_name == _FAITHFULNESS_NAME:
                  if answer == _YES_VALUE:
                      answer_value = 1
                  elif answer == _NO_VALUE:
                      answer_value = 0
                  else:
                      raise ValueError(
                          f"Unknown answer to {_FAITHFULNESS_NAME}: {answer}"
                      )
              else:
                  answer_value = (int(answer) - 1) / 4
              stats[answer_name].add(answer_value)
      return list(stats.values())
  
file_path: H2O/h2o_hf/helm/src/helm/benchmark/metrics/summarization_critique_metrics.py
imports: from typing import Dict, List
  from helm.benchmark.adaptation.request_state import RequestState
  from helm.benchmark.adaptation.adapter_spec import AdapterSpec
  from .metric import Metric
  from .metric_name import MetricName
  from .metric_service import MetricService
  from .statistic import Stat
  from helm.common.critique_request import CritiqueTaskTemplate, CritiqueQuestionTemplate, CritiqueRequest, QuestionType
label_desc: Use image node_id 3 for calling the SummarizationCritiqueMetric obj's underlying member method code with example usage: obj.evaluate_generation(adapter_spec, request_state, metric_service, eval_cache_path) and returns: list, list, list