class_docstr: Dataset evaluator.
class_docstr_tok: ["Dataset","evaluator","."]
docstr: Calculate perplexity.
docstr_tok: ["Calculate","perplexity","."]
code: def _calculate_loss_metrics(self, metric: str, category: str):
      """Calculate perplexity."""
      if metric == "perplexity":
          weight = (
              len(self.data[category]["data"])
              / self.metric_total_length[metric]
          )
          losses = [
              min(sample["loss"])
              for sample in self.data[category]["data"]
          ]
          perplexity = np.mean(np.exp(np.array(losses)))
  
          self.evaluation_results["perplexity"][category] = (
              perplexity,
              len(self.data[category]["data"]),
          )
          self.evaluation_results["perplexity"]["ALL"] += (
              perplexity * weight
          )
      elif metric == "ppl_score":
          weight = (
              len(self.data[category]["data"])
              / self.metric_total_length[metric]
          )
          losses = [
              min(sample["loss"])
              for sample in self.data[category]["data"]
          ]
          perplexity_score = np.mean(np.exp(-np.array(losses))) * 100
  
          self.evaluation_results["ppl_score"][category] = (
              perplexity_score,
              len(self.data[category]["data"]),
          )
          self.evaluation_results["ppl_score"]["ALL"] += (
              perplexity_score * weight
          )
      elif (
          metric == "ppl_score_over_choices"
          and self.data[category]["inference_kwargs"]["all_classes"]
          is not None
      ):
          weight = (
              len(self.data[category]["data"])
              / self.metric_total_length[metric]
          )
          loss_over_choices = [
              sample["loss_over_choices"]
              for sample in self.data[category]["data"]
          ]
          perplexity_score_over_choices = (
              np.mean(np.exp(-np.array(loss_over_choices))) * 100
          )
  
          self.evaluation_results["ppl_score_over_choices"][
              category
          ] = (
              perplexity_score_over_choices,
              len(self.data[category]["data"]),
          )
          self.evaluation_results["ppl_score_over_choices"]["ALL"] += (
              perplexity_score_over_choices * weight
          )
      elif metric == "per_byte_perplexity":
          weight = (
              len(self.data[category]["data"])
              / self.metric_total_length[metric]
          )
          losses = [
              min(sample["loss_sum"])
              for sample in self.data[category]["data"]
          ]
          perplexity = np.mean(
              np.exp(
                  np.array(losses) / np.array(self.N_bytes[category])
              )
          )
  
          self.evaluation_results["per_byte_perplexity"][
              category
          ] = perplexity
          self.evaluation_results["per_byte_perplexity"]["ALL"] += (
              perplexity * weight
          )
      elif metric == "per_byte_ppl_score":
          weight = (
              len(self.data[category]["data"])
              / self.metric_total_length[metric]
          )
          losses = [
              min(sample["loss_sum"])
              for sample in self.data[category]["data"]
          ]
          perplexity_score = (
              np.mean(
                  np.exp(
                      -np.array(losses)
                      / np.array(self.N_bytes[category])
                  )
              )
              * 100
          )
  
          self.evaluation_results["per_byte_ppl_score"][
              category
          ] = perplexity_score
          self.evaluation_results["per_byte_ppl_score"]["ALL"] += (
              perplexity_score * weight
          )
      elif metric == "loss_over_all_tokens":
          weight = (
              len(self.data[category]["data"])
              / self.metric_total_length[metric]
          )
          losses = [
              min(sample["loss_sum"])
              for sample in self.data[category]["data"]
          ]
          token_nums = [
              sample["token_num"][np.argmin(sample["loss_sum"])]
              for sample in self.data[category]["data"]
          ]
          perplexity = np.sum(np.array(losses)) / np.sum(
              np.array(token_nums)
          )
  
          self.evaluation_results["loss_over_all_tokens"][
              category
          ] = perplexity
          self.evaluation_results["loss_over_all_tokens"]["ALL"] += (
              perplexity * weight
          )
  
          # The number of tokens can be used for normalizing.
          # See https://github.com/SkyworkAI/Skywork/issues/43#issuecomment-1811733834
          print(
              f"{self.model_name} {category} token num: {np.sum(np.array(token_nums))}"
          )
  
file_path: ColossalAI/applications/ColossalEval/colossal_eval/evaluate/dataset_evaluator/dataset_evaluator.py
imports: import os
  from typing import Dict, List, Union
  import colossal_eval.evaluate.dataset_evaluator.metrics
  import numpy
  import tqdm
  from colossal_eval.utils import jdump
  import colossal_eval.evaluate.dataset_evaluator.gpt_judge
label_desc: Use image node_id 6 for calling the DatasetEvaluator obj's underlying member method code with example usage: obj._calculate_loss_metrics(metric, category) without return types