code: def _test_grad_acc(self, model, batch_dim, use_no_sync_context=True):
      make_cudnn_deterministic()
      # Generate two input batches. We'll test that we get the same grads if
      # we train on them sequentially while accumulating grads (with no_sync
      # or without no_sync) vs. concatenating the batches and training in one go.
      #
      # The difference between with no_sync and without is GPU memory vs. networking
      # bandwidth tradeoff.
      batch1 = model.module.get_input(torch.device("cuda"))
      assert isinstance(batch1, tuple)
      batch2 = tuple(
          # This randomly permutes the values in a multi-dim tensor.
          x.view(-1)[torch.randperm(x.numel())].view_as(x)
          for x in batch1
      )
      for x, y in zip(batch1, batch2):
          assert not torch.all(x == y)
  
      # Concat the batches along batch dimension.
      concat_batch = tuple(
          torch.cat((x, y), dim=batch_dim)
          for (x, y) in zip(batch1, batch2)
      )
  
      # Establish reference behavior on the concat batch.
      model.zero_grad()
      output = model(*concat_batch)
      ref_loss = model.module.get_loss(concat_batch, output)
      ref_loss.backward()
      ref_grads = [p.grad.detach().clone() for p in model.parameters()]
  
      # Test that we get the same results by accumulating grads.
      model.zero_grad()
      context = contextlib.suppress()
      if use_no_sync_context:
          context = model.no_sync()
      with context:  # accumulate gradients from the first batch
          output = model(*batch1)
          loss1 = model.module.get_loss(batch1, output)
          loss1.backward()
      output = model(*batch2)
      loss2 = model.module.get_loss(batch2, output)
      loss2.backward()
      accumulated_loss = loss1 + loss2
      accumulated_grads = [
          p.grad.detach().clone() for p in model.parameters()
      ]
  
      torch.testing.assert_allclose(ref_loss, accumulated_loss)
      assert objects_are_equal(
          ref_grads, accumulated_grads, raise_exception=True
      )
  
imports: import contextlib
  import functools
  import itertools
  import unittest
  from unittest.mock import patch
  from parameterized import parameterized
  import torch
  from fairscale.fair_dev.testing.testing import DummyProcessGroup, make_cudnn_deterministic, objects_are_equal
  from fairscale.nn.data_parallel import FullyShardedDataParallel
  from .test_fsdp import DistributedTest, NestedWrappedModule, rename_test, spawn_and_init
label_desc: Use image node_id 8 for calling the TestGradAcc obj's underlying member method code with example usage: obj._test_grad_acc(model, batch_dim, use_no_sync_context) without return types