docstr: Tests basic functionality of HYBRID_SHARD and _HYBRID_SHARD_ZERO2:
      1. Inter and intra-node process groups are correctly setup
      2. Process groups are the same across FSDP wrapped instances
      3. reduce_scatter and allreduce called the expected no. of times
docstr_tok: ["Tests","basic","functionality","of","HYBRID_SHARD","and","_HYBRID_SHARD_ZERO2",":","1",".","Inter","and","intra-node","process","groups","are","correctly","setup","2",".","Process","groups","are","the","same","across","FSDP","wrapped","instances","3.","reduce_scatter","and","allreduce","called","the","expected","no",".","of","times"]
code: def test_fsdp_hybrid_shard_basic_setup(self):
      """
      Tests basic functionality of HYBRID_SHARD and _HYBRID_SHARD_ZERO2:
          1. Inter and intra-node process groups are correctly setup
          2. Process groups are the same across FSDP wrapped instances
          3. reduce_scatter and allreduce called the expected no. of times
      """
      self.run_subtests(
          {
              "hsdp_sharding_strategy": [
                  ShardingStrategy.HYBRID_SHARD,
                  ShardingStrategy._HYBRID_SHARD_ZERO2,
              ],
              "sharding_strategy_mode": [
                  ShardingStrategyMode.ALL_HYBRID_SHARD,
                  ShardingStrategyMode.MIXED_HYBRID_FULL_SHARD,
              ],
              "use_orig_params": [False, True],
          },
          self._test_fsdp_hybrid_shard_basic_setup,
      )
  
imports: import contextlib
  import sys
  from collections import Counter
  from enum import auto, Enum
  from functools import partial
  from typing import List, Optional, Tuple
  import torch
  import torch.distributed
  import torch.distributed.fsdp._traversal_utils
  import torch.nn
  from torch.distributed.distributed_c10d import _rank_not_in_group
  from torch.distributed.fsdp import FullyShardedDataParallel, ShardingStrategy, StateDictType
  from torch.distributed.fsdp._init_utils import _init_intra_and_inter_node_groups, HYBRID_SHARDING_STRATEGIES
  from torch.distributed.fsdp.wrap import ModuleWrapPolicy
  from torch.nn import TransformerDecoderLayer, TransformerEncoderLayer
  from torch.testing._internal.common_distributed import skip_if_lt_x_gpu
  from torch.testing._internal.common_fsdp import CUDAInitMode, FSDPInitMode, FSDPTest, TransformerWithSharedParams
  from torch.testing._internal.common_utils import instantiate_parametrized_tests, run_tests, TEST_WITH_DEV_DBG_ASAN
label_desc: Use image node_id 8 for calling the TestFSDPHybridShard obj's underlying member method code with example usage: obj.test_fsdp_hybrid_shard_basic_setup() without return types