docstr: Move a file from one directory to another, possibly across filesystems.
  
  This implementation specifically addresses the following issue with distributed training:
  
  1. The `save_path` is a directory local to the node, in which case every node should write
     checkpoints separately.
  2. The `save_path` is a remote / global filesystem like NFS, in which case only the coordinator
     should write checkpoints.
code: def safe_move_file(self, src: str, dst: str):
      """Move a file from one directory to another, possibly across filesystems.
  
      This implementation specifically addresses the following issue with distributed training:
  
      1. The `save_path` is a directory local to the node, in which case every node should write
         checkpoints separately.
      2. The `save_path` is a remote / global filesystem like NFS, in which case only the coordinator
         should write checkpoints.
      """
      try:
          os.replace(src, dst)
      except OSError as err:
          if err.errno == errno.EXDEV:
              # Tried to move to an external filesystem. This means we should only run this on the coordinator
              if not self.distributed.is_coordinator():
                  logger.info(
                      f"Skipping writing checkpoint from rank {self.distributed.rank()} as it is not the coordinator "
                      f"and the destination filesystem is remote."
                  )
                  return
  
              # Generate a unique ID, and copy `<src>` to the target directory with a temporary name `<dst>.<ID>.tmp`.
              # Because we're copying across a filesystem boundary, this initial copy may not be atomic.  We insert a
              # random UUID so if different processes are copying into `<dst>`, they don't overlap in their tmp
              # copies.
              copy_id = uuid.uuid4()
              tmp_dst = f"{dst}.{copy_id}.tmp"
              shutil.copyfile(src, tmp_dst)
  
              # Atomic replace file onto the new name, and clean up original source file.
              os.replace(tmp_dst, dst)
              os.unlink(src)
          else:
              raise
  
imports: import errno
  import logging
  import os
  import re
  import shutil
  import signal
  import tempfile
  import uuid
  from abc import ABC, abstractmethod
  from glob import glob
  from typing import Any, Dict, Mapping, Optional, Tuple, TYPE_CHECKING
  import torch
  from torch.optim import Optimizer
  from ludwig.api_annotations import DeveloperAPI
  from ludwig.modules.lr_scheduler import LRScheduler
label_desc: Use image node_id 7 for calling the MultiNodeCheckpoint obj's underlying member method code with example usage: obj.safe_move_file(src, dst) without return types