code: def pretrain(self, pretrain_dir: str) -> None:
      device_mesh = init_device_mesh(
          self.device_type, (self.world_size,)
      )
  
      model = PreTrainedModel().cuda()
      model = FSDP(model, device_mesh=device_mesh)
      optim = torch.optim.Adam(model.parameters(), lr=1e-3)
  
      # Training
      for i in range(3):
          batch = torch.rand(32, DIM, device="cuda")
          loss = model(batch).sum()
          loss.backward()
          optim.step()
          optim.zero_grad()
  
      # Save state_dict
      model_state_dict, optim_state_dict = get_state_dict(
          model, optimizers=optim
      )
      saved_state_dict = {
          "model": model_state_dict,
          "optim": optim_state_dict,
      }
      dist_cp.save_state_dict(
          state_dict=saved_state_dict,
          storage_writer=dist_cp.FileSystemWriter(pretrain_dir),
      )
  
imports: import os
  import sys
  import torch
  import torch.distributed
  import torch.distributed.checkpoint
  import torch.nn
  from torch.distributed._tensor import init_device_mesh
  from torch.distributed.checkpoint.state_dict import get_state_dict, set_state_dict, StateDictOptions
  from torch.distributed.fsdp import FullyShardedDataParallel
  from torch.testing._internal.common_distributed import skip_if_lt_x_gpu
  from torch.testing._internal.common_utils import run_tests, TEST_WITH_DEV_DBG_ASAN
  from torch.testing._internal.distributed._tensor.common_dtensor import DTensorTestBase, with_comms
  from torch.testing._internal.distributed.checkpoint_utils import with_temp_dir
label_desc: Use image node_id 3 for calling the TestFineTuning obj's underlying member method code with example usage: obj.pretrain(pretrain_dir) without return types