'Args:
devices (list[int]): physical GPU ids. If None, will use CUDA_VISIBLE_DEVICES'

def __init__(self, devices=None):

    if (devices is None):
        env = os.environ.get('CUDA_VISIBLE_DEVICES')
        assert (env is not None), '[GPUUtilizationTracker]    Both    devices    and    CUDA_VISIBLE_DEVICES    are    None!'
        self._devices = env.split(',')
    else:
        self._devices = list(map(str, devices))
    assert len(self._devices), '[GPUUtilizationTracker]    No    GPU    device    given!'
    self._command = ('nvidia-smi    --query-gpu=utilization.gpu    --format=csv,noheader,nounits    -i    ' + ','.join(self._devices))
    (_, ret) = subproc_call(self._command)
    assert (ret == 0), 'Cannot    fetch    GPU    utilization!'
