class_docstr: Group of timers. Borrowed from Nvidia Megatron code
class_docstr_tok: ["Group","of","timers",".","Borrowed","from","Nvidia","Megatron","code"]
code: def memory_usage():
      alloc = "mem_allocated: {:.4f} GB".format(
          torch.cuda.memory_allocated() / (1024 * 1024 * 1024)
      )
      max_alloc = "max_mem_allocated: {:.4f} GB".format(
          torch.cuda.max_memory_allocated() / (1024 * 1024 * 1024)
      )
      cache = "cache_allocated: {:.4f} GB".format(
          torch.cuda.memory_cached() / (1024 * 1024 * 1024)
      )
      max_cache = "max_cache_allocated: {:.4f} GB".format(
          torch.cuda.max_memory_cached() / (1024 * 1024 * 1024)
      )
      return " | {} | {} | {} | {}".format(
          alloc, max_alloc, cache, max_cache
      )
  
imports: import time
  import torch
  from numpy import mean
  from deepspeed.utils.logging import log_dist
  from deepspeed import comm
label_desc: Use image node_id 4 for calling the SynchronizedWallClockTimer obj's underlying member method code with example usage: obj.memory_usage() and returns: str