code: def rmtree_nfs_safe(
      path: Union[str, os.PathLike],
      ignore_errors: bool = False,
      onerror: Optional[Callable[[Any, Any, Any], Any]] = None,
  ) -> None:
      """
      Reported shutil.rmtree() fails sometimes under NFS, and apparently the right solution is to
      just retry a few times.
      """
      max_tries = 5
      for tries in range(1, max_tries + 1):
          try:
              return shutil.rmtree(
                  path, ignore_errors=ignore_errors, onerror=onerror
              )
          except OSError as e:
              if e.errno != errno.ENOTEMPTY or tries == max_tries:
                  raise
              # This should not be possible, since rmtree empties directories before rmdir'ing them.
              logger.debug(
                  f"rmtree() failed ({e}), is this NFS?  Retrying (tries={tries})..."
              )
              # All 5 tries should take on the order of half a second.
              time.sleep(2**tries / 100)
  
imports: import collections
  import contextlib
  import copy
  import datetime
  import enum
  import errno
  import inspect
  import json
  import logging
  import math
  import numbers
  import os
  import pathlib
  import random
  import re
  import shutil
  import signal
  import socket
  import stat
  import subprocess
  import time
  import uuid
  from typing import Any, Callable, Dict, Iterator, List, Optional, Set, SupportsFloat, Tuple, Union, cast
  import psutil
  import determined
  from determined import constants
  from determined.common import check, detignore, util
label_desc: Use image node_id 1 for calling a global function with example usage: rmtree_nfs_safe(path, ignore_errors, onerror) and returns: shutil