


import operator as op


__snake_case            =   "scaler.pt"
__snake_case            =   "pytorch_model"
__snake_case            =   "random_states"
__snake_case            =   "optimizer"
__snake_case            =   "scheduler"
__snake_case            =   "pytorch_model.bin"
__snake_case            =   "pytorch_model.bin.index.json"
__snake_case            =   "model.safetensors"
__snake_case            =   "model.safetensors.index.json"
__snake_case            =   "1.10.2"
__snake_case            =   "py38"
__snake_case            =   "4.17.0"
__snake_case            =   ["ml.p3.16xlarge", "ml.p3dn.24xlarge", "ml.p4dn.24xlarge"]
__snake_case            =   ["FULL_SHARD", "SHARD_GRAD_OP", "NO_SHARD", "HYBRID_SHARD", "HYBRID_SHARD_ZERO2"]
__snake_case            =   ["TRANSFORMER_BASED_WRAP", "SIZE_BASED_WRAP", "NO_WRAP"]
__snake_case            =   ["BACKWARD_PRE", "BACKWARD_POST", "NO_PREFETCH"]
__snake_case            =   ["FULL_STATE_DICT", "LOCAL_STATE_DICT", "SHARDED_STATE_DICT"]
__snake_case            =   "2.0.1"
__snake_case            =   ["pdsh", "standard", "openmpi", "mvapich"]
__snake_case            =   ["default", "reduce-overhead", "max-autotune"]

__snake_case            =   {">": op.gt, ">=": op.ge, "==": op.eq, "!=": op.ne, "<=": op.le, "<": op.lt}

# These are the args for `torch.distributed.launch` for pytorch < 1.9
__snake_case            =   [
    "nnodes",
    "nproc_per_node",
    "rdzv_backend",
    "rdzv_endpoint",
    "rdzv_id",
    "rdzv_conf",
    "standalone",
    "max_restarts",
    "monitor_interval",
    "start_method",
    "role",
    "module",
    "m",
    "no_python",
    "run_path",
    "log_dir",
    "r",
    "redirects",
    "t",
    "tee",
    "node_rank",
    "master_addr",
    "master_port",
]

__snake_case            =   ["DEEPSPEED", "MULTI_GPU", "FSDP", "MEGATRON_LM"]
__snake_case            =   ["DEEPSPEED", "MULTI_XPU", "FSDP"]








import functools
import gc
import inspect

import torch

from .imports import is_npu_available, is_xpu_available






def 							_lowercase    (		*SCREAMING_SNAKE_CASE_   :		List[Any]	):





			"""simple docstring"""




			if not isinstance(SCREAMING_SNAKE_CASE_       ,  SCREAMING_SNAKE_CASE_	):
						UpperCamelCase         =					list(SCREAMING_SNAKE_CASE_	)
			for i in range(len(SCREAMING_SNAKE_CASE_	)	):
						UpperCamelCase         =					None
			gc.collect()
			if is_xpu_available():
						torch.xpu.empty_cache()
			elif is_npu_available():
						torch.npu.empty_cache()
			else:
						torch.cuda.empty_cache()
			return objects






def 							_lowercase    (		SCREAMING_SNAKE_CASE_   :		Exception	):





			"""simple docstring"""




			UpperCamelCase         =					[
			    """CUDA out of memory.""",  # CUDA OOM
			    """cuDNN error: CUDNN_STATUS_NOT_SUPPORTED.""",  # CUDNN SNAFU
			    """DefaultCPUAllocator: can't allocate memory""",  # CPU OOM
			]
			if isinstance(SCREAMING_SNAKE_CASE_       ,  SCREAMING_SNAKE_CASE_	) and len(exception.args	) == 1:
						return any(err in exception.args[0] for err in _statements	)
			return False






def 							_lowercase    (		SCREAMING_SNAKE_CASE_   :		callable = None       ,  SCREAMING_SNAKE_CASE_   :		int = 128	):





			"""simple docstring"""




			if function is None:
						return functools.partial(SCREAMING_SNAKE_CASE_       ,  starting_batch_size=SCREAMING_SNAKE_CASE_	)

			UpperCamelCase         =					starting_batch_size

			def decorator(*SCREAMING_SNAKE_CASE_   :		Union[str, Any]       ,  **SCREAMING_SNAKE_CASE_   :		Tuple	):
						nonlocal batch_size
						gc.collect()
						if is_xpu_available():
									torch.xpu.empty_cache()
						elif is_npu_available():
									torch.npu.empty_cache()
						else:
									torch.cuda.empty_cache()
						UpperCamelCase         =					list(inspect.signature(SCREAMING_SNAKE_CASE_	).parameters.keys()	)
						# Guard against user error
						if len(SCREAMING_SNAKE_CASE_	) < (len(SCREAMING_SNAKE_CASE_	) + 1):
									UpperCamelCase         =					""", """.join([f'{arg}={value}' for arg, value in zip(params[1:]       ,  args[1:]	)]	)
									raise TypeError(
									    f'Batch size was passed into `{function.__name__}` as the first argument when called.'
									    f'Remove this as the decorator already does so: `{function.__name__}({arg_str})`'	)
						while True:
									if batch_size == 0:
												raise RuntimeError("""No executable batch size found, reached zero."""	)
									try:
												return function(SCREAMING_SNAKE_CASE_       ,  *SCREAMING_SNAKE_CASE_       ,  **SCREAMING_SNAKE_CASE_	)
									except Exception as e:
												if should_reduce_batch_size(SCREAMING_SNAKE_CASE_	):
															gc.collect()
															if is_xpu_available():
																		torch.xpu.empty_cache()
															elif is_npu_available():
																		torch.npu.empty_cache()
															else:
																		torch.cuda.empty_cache()
															batch_size //= 2
												else:
															raise

			return decorator




