class_docstr: ## Sequential module for `Zero3Layer` layers
class_docstr_tok: ["#","#","Sequential","module","for","`","Zero3Layer","`","layers"]
docstr: :param modules: List of `Zero3Layer` layers
docstr_tok: [":","param","modules",":","List","of","`","Zero3Layer","`","layers"]
code: def __init__(self, modules: List[Zero3Layer]):
      """
      :param modules: List of `Zero3Layer` layers
      """
      super().__init__()
  
      # CUDA stream to fetch parameters
      self.fetch_stream = torch.cuda.Stream()
      # CUDA stream to back up (accumulate) gradients
      self.backup_stream = torch.cuda.Stream()
  
      # Set the streams and preceding and proceeding layers for each `Zero3Layer` layer
      for i in range(len(modules)):
          # Set layer index
          modules[i].layer_idx = i
          # Set streams
          modules[i].fetch_stream = self.fetch_stream
          modules[i].backup_stream = self.backup_stream
          # Set proceeding layers
          if i + 1 < len(modules):
              modules[i].next_layer.append(modules[i + 1])
          # Set preceding layers
          if i - 1 >= 0:
              modules[i].prev_layer.append(modules[i - 1])
  
      # Store list of modules
      self.module_list = nn.ModuleList(modules)
  
imports: import functools
  from typing import List, Optional, Tuple
  import torch
  import torch.distributed
  from torch import nn
label_desc: Use image node_id 1 to create a new Zero3Sequential object from inherited base classes: nn with example: obj = Zero3Sequential(modules)