code: def compare_stats() -> None:
      if not fetch_master_log():
          print("Skip compare stats because error at fetch master")
          return
      gpu_from_log, global_start, global_end = parse_log_for_gpu_stats(
          log_path
      )
      try:
          res = bindings.get_ResourceAllocationRaw(
              api_utils.determined_test_session(),
              timestampAfter=global_start,
              timestampBefore=global_end,
          )
      except api.errors.ForbiddenException:
          print(
              "Skip compare stats because missing access to resource allocations"
          )
          return
      gpu_from_api = 0
      gpu_from_api_map = {}
      instance_from_api = 0
      instance_from_api_map = {}
      for r in (res.to_json())["resourceEntries"]:
          if r["kind"] == "agent" and r["seconds"] > 0:
              gpu_from_api += r["seconds"]
              if r["username"] not in gpu_from_api_map:
                  gpu_from_api_map[r["username"]] = 0
              gpu_from_api_map[r["username"]] += r["seconds"]
          if r["kind"] == "instance" and r["seconds"] > 0:
              instance_from_api += r["seconds"]
              if r["username"] not in instance_from_api_map:
                  instance_from_api_map[r["username"]] = 0
              instance_from_api_map[r["username"]] += r["seconds"]
      for ins in instance_from_api_map:
          # make sure instance initialization time is less than 5 mins
          if ins in gpu_from_api_map:
              assert (
                  instance_from_api_map[ins] - gpu_from_api_map[ins]
                  < 60 * 5
              )
  
      print(f"Agent time: logs={gpu_from_log}, api={gpu_from_api}")
      # make sure agent stats get from script is less than 5% difference with those get from api
      max_diff = max(gpu_from_api, gpu_from_log) * 0.05
      # For shorter tests, a tolerance that is percentage based can make unreasonably tight timing.
      max_diff = max(max_diff, 3)
      # slurm and pbs needs looser tolerances
      if api_utils.is_hpc():
          max_diff = max(max_diff, 10)
      assert abs(gpu_from_log - gpu_from_api) <= max_diff
  
file_path: determined/e2e_tests/tests/nightly/compute_stats.py
imports: import re
  import subprocess
  import traceback
  from datetime import datetime, timedelta, timezone
  from typing import Any, Dict, Tuple
  from dateutil import parser
  from determined.common import api
  from determined.common.api import bindings
  from tests import api_utils
  from tests import config
label_desc: Use image node_id 4 for calling a global function with example usage: compare_stats() without return types