code: def update_scale_hysteresis_body(
      self,
      init_scale,
      growth_factor,
      backoff_factor,
      growth_interval,
      hysteresis,
  ):
      scale_ref = float(init_scale)
      grow_tracker_ref = 0
      hysteresis_tracker_ref = 0
  
      scale = torch.tensor(
          [init_scale], dtype=torch.float32, device="cuda"
      )
      growth_tracker = torch.tensor(
          [0], dtype=torch.int32, device="cuda"
      )
      hysteresis_tracker = torch.tensor(
          [hysteresis], dtype=torch.int32, device="cuda"
      )
  
      # Infs appear for hysteresis-1 iterations, scale shouldn't change
      found_inf = torch.tensor([1], dtype=torch.float32, device="cuda")
      for i in range(hysteresis - 1):
          update_scale_hysteresis(
              scale,
              growth_tracker,
              hysteresis_tracker,
              found_inf,
              growth_factor,
              backoff_factor,
              growth_interval,
              hysteresis,
          )
      self.assertTrue(scale.item() == init_scale)
  
      # No infs for growth_interval-1 iterations, scale shouldn't change
      found_inf.zero_()
      for i in range(growth_interval - 1):
          update_scale_hysteresis(
              scale,
              growth_tracker,
              hysteresis_tracker,
              found_inf,
              growth_factor,
              backoff_factor,
              growth_interval,
              hysteresis,
          )
      self.assertTrue(scale.item() == init_scale)
  
      # Infs appear for more than hysteresis iterations, scale should be backed off
      found_inf.fill_(1)
      extra_iters = random.randint(0, 1000)
      scale_before = scale.detach().item()
      scale_ref = scale_before
      for i in range(hysteresis + extra_iters):
          update_scale_hysteresis(
              scale,
              growth_tracker,
              hysteresis_tracker,
              found_inf,
              growth_factor,
              backoff_factor,
              growth_interval,
              hysteresis,
          )
      for i in range(1 + extra_iters):
          # Scale is continuously backed off for each iteration with an inf
          scale_new = scale_ref * backoff_factor
          if isfinite(scale_new):
              scale_ref = scale_new
          else:
              scale_ref = 0  # Scale update kernel does not check for underflow when backing off, which results in zero
      self.assertTrue(scale.item() == scale_ref)
  
      # No infs for more than growth_interval iterations, scale should be increased
      found_inf.fill_(0)
      extra_iters = random.randint(0, 1000)
      scale_before = scale.detach().item()
      scale_ref = scale_before
      for i in range(growth_interval + extra_iters):
          update_scale_hysteresis(
              scale,
              growth_tracker,
              hysteresis_tracker,
              found_inf,
              growth_factor,
              backoff_factor,
              growth_interval,
              hysteresis,
          )
      for i in range(
          1 + int(math.floor(extra_iters / growth_interval))
      ):
          # Scale is grown every growth_interval iterations
          scale_new = scale_ref * growth_factor
          if isfinite(scale_new):
              scale_ref = scale_new
      self.assertTrue(scale.item() == scale_ref)
  
file_path: apex/tests/L0/run_amp/test_update_scale_hysteresis.py
imports: import unittest
  import random
  import math
  import torch
label_desc: Use image node_id 3 for calling the TestUpdateScaleHysteresis obj's underlying member method code with example usage: obj.update_scale_hysteresis_body(init_scale, growth_factor, backoff_factor, growth_interval, hysteresis) without return types