class_docstr: Summarize the average GPU utilization within an epoch.
  
  It will start a process to obtain GPU utilization through NVML every second
  within the epoch (the trigger_epoch time was not included),
  and write average utilization to monitors.
  
  This callback creates a process, therefore it's not safe to be used with MPI.
class_docstr_tok: ["Summarize","the","average","GPU","utilization","within","an","epoch",".","It","will","start","a","process","to","obtain","GPU","utilization","through","NVML","every","second","within","the","epoch","(","the","trigger_epoch","time","was","not","included",")",",","and","write","average","utilization","to","monitors",".","This","callback","creates","a","process",",","therefore","it","'s","not","safe","to","be","used","with","MPI","."]
docstr: Args:
      devices (list[int])
code: def worker(evt, rst_queue, stop_evt, devices):
      """
      Args:
          devices (list[int])
      """
      try:
          with NVMLContext() as ctx:
              devices = [ctx.device(i) for i in devices]
              while True:
                  evt.wait()  # start epoch
                  evt.clear()
                  if stop_evt.is_set():  # or on exit
                      return
  
                  stats = np.zeros((len(devices),), dtype="f4")
                  cnt = 0
                  while True:
                      time.sleep(1)
  
                      data = [d.utilization()["gpu"] for d in devices]
                      data = list(map(float, data))
                      stats += data
                      cnt += 1
  
                      if evt.is_set():  # stop epoch
                          if stop_evt.is_set():  # or on exit
                              return
                          if cnt > 1:
                              # Ignore the last datapoint. Usually is zero, makes us underestimate the util.
                              stats -= data
                              cnt -= 1
                          rst_queue.put(stats / cnt)
                          evt.clear()
                          break
      except Exception:
          logger.exception("Exception in GPUUtilizationTracker.worker")
          rst_queue.put(-1)
          return
  
imports: import multiprocessing
  import numpy
  import os
  import time
  import tensorflow
  from six.moves import map, queue
  import psutil
  from ..tfutils.common import gpu_available_in_session
  from ..utils import logger
  from ..utils.timer import Timer
  from ..utils.concurrency import ensure_proc_terminate, start_proc_mask_signal
  from ..utils.gpu import get_num_gpu
  from ..utils.nvml import NVMLContext
  from .base import Callback
label_desc: Use image node_id 9 for calling the GPUUtilizationTracker obj's underlying member method code with example usage: obj.worker(evt, rst_queue, stop_evt, devices) without return types