class_docstr: Dataset evaluator.
class_docstr_tok: ["Dataset","evaluator","."]
docstr: Calculate label-based metrics.
docstr_tok: ["Calculate","label-based","metrics","."]
code: def _calculate_label_metrics(self, metric: str, category: str):
      """Calculate label-based metrics."""
      weight = (
          len(self.data[category]["data"])
          / self.metric_total_length[metric]
      )
  
      str_label_map = {
          choice: idx
          for idx, choice in enumerate(
              self.data[category]["inference_kwargs"]["all_classes"]
          )
      }
  
      references = [
          str_label_map[sample["target"]]
          for sample in self.data[category]["data"]
      ]
      [sample["output"] for sample in self.data[category]["data"]]
  
      flag = False
      logits = []
      for i, sample in enumerate(self.data[category]["data"]):
          if np.any(
              np.isnan(
                  np.array(list(sample["logits_over_choices"].values()))
              )
          ):
              if not flag:
                  print(
                      f"NaN in the logits, switch to exact match for category {category} in dataset {self.dataset_name} in model {self.model_name}."
                  )
                  flag = True
              score = 0
              for ref in sample["target"]:
                  score = max(
                      score,
                      metric_helper.single_choice_accuracy(
                          sample["output"],
                          ref,
                          all_classes=self.data[category][
                              "inference_kwargs"
                          ]["all_classes"],
                      ),
                  )
  
                  score = max(
                      score,
                      metric_helper.accuracy_by_options(
                          sample["input"], sample["output"], ref
                      ),
                  )
              logits.append(references[i] if score == 1 else -1)
          else:
              logits.append(
                  np.argmax(
                      np.array(
                          list(sample["logits_over_choices"].values())
                      )
                  )
              )
  
      references = np.array(references)
      logits = np.array(logits)
      scores = (
          np.sum(references == logits)
          / len(self.data[category]["data"])
          * 100
      )
  
      self.evaluation_results[metric][category] = (
          scores,
          len(self.data[category]["data"]),
      )
      self.evaluation_results[metric]["ALL"] += scores * weight
  
file_path: colossalai/applications/ColossalEval/colossal_eval/evaluate/dataset_evaluator/dataset_evaluator.py
imports: import os
  from typing import Dict, List, Union
  import colossal_eval.evaluate.dataset_evaluator.metrics
  import numpy
  import tqdm
  from colossal_eval.utils import jdump
  import colossal_eval.evaluate.dataset_evaluator.gpt_judge
label_desc: Use image node_id 2 for calling the DatasetEvaluator obj's underlying member method code with example usage: obj._calculate_label_metrics(metric, category) without return types