class_docstr: Dataset evaluator.
class_docstr_tok: ["Dataset","evaluator","."]
docstr: Calculate combined metrics.
docstr_tok: ["Calculate","combined","metrics","."]
code: def _calculate_combined_metrics(self, metric: str, category: str):
      """Calculate combined metrics."""
      weight = (
          len(self.data[category]["data"])
          / self.metric_total_length[metric]
      )
  
      references = [
          sample["target"] for sample in self.data[category]["data"]
      ]
      predictions = [
          sample["output"] for sample in self.data[category]["data"]
      ]
  
      str_label_map = {
          choice: idx
          for idx, choice in enumerate(
              self.data[category]["inference_kwargs"]["all_classes"]
          )
      }
  
      references_labels = [
          str_label_map[sample["target"][0]]
          for sample in self.data[category]["data"]
      ]
      predictions = [
          sample["output"] for sample in self.data[category]["data"]
      ]
  
      flag = False
      logits = []
      for i, sample in enumerate(self.data[category]["data"]):
          if np.any(
              np.isnan(
                  np.array(list(sample["logits_over_choices"].values()))
              )
          ):
              if not flag:
                  print(
                      f"NaN in the logits, switch to exact match for category {category} in dataset {self.dataset_name} in model {self.model_name}."
                  )
                  flag = True
              score = 0
              for ref in sample["target"]:
                  score = max(
                      score,
                      metric_helper.single_choice_accuracy(
                          sample["output"],
                          ref,
                          all_classes=self.data[category][
                              "inference_kwargs"
                          ]["all_classes"],
                      ),
                  )
              logits.append(references[i] if score == 1 else -1)
          else:
              logits.append(
                  np.argmax(
                      np.array(
                          list(sample["logits_over_choices"].values())
                      )
                  )
              )
  
      metric_method = eval("metric_helper." + metric)
  
      total_score = 0.0
      for prediction, reference, references_label, softmax in zip(
          predictions, references, references_labels, logits
      ):
          score = 0.0
  
          for ref in reference:
              score = max(
                  score,
                  metric_method(
                      prediction,
                      ref,
                      all_classes=self.data[category][
                          "inference_kwargs"
                      ]["all_classes"],
                  ),
              )
          if references_label == softmax:
              score = 1
  
          total_score += score
      total_score = total_score * 100 / len(self.data[category]["data"])
  
      self.evaluation_results[metric][category] = (
          total_score,
          len(self.data[category]["data"]),
      )
      self.evaluation_results[metric]["ALL"] += total_score * weight
  
file_path: ColossalAI/applications/ColossalEval/colossal_eval/evaluate/dataset_evaluator/dataset_evaluator.py
imports: import os
  from typing import Dict, List, Union
  import colossal_eval.evaluate.dataset_evaluator.metrics
  import numpy
  import tqdm
  from colossal_eval.utils import jdump
  import colossal_eval.evaluate.dataset_evaluator.gpt_judge
label_desc: Use image node_id 3 for calling the DatasetEvaluator obj's underlying member method code with example usage: obj._calculate_combined_metrics(metric, category) without return types