


'''simple docstring'''



import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


__A	=logging.get_logger(__name__)







def     _UpperCamelCase      (		):
				# Get the sagemaker specific mp parameters from smp_options variable.
				UpperCAmelCase__  :					Any        =     os.getenv("""SM_HP_MP_PARAMETERS""" ,      """{}"""					)
				try:
								# Parse it and check the field "partitions" is included, it is required for model parallel.
								UpperCAmelCase__  :					Tuple        =     json.loads(UpperCamelCase__					)
								if "partitions" not in smp_options:
												return False
				except json.JSONDecodeError:
								return False

				# Get the sagemaker specific framework parameters from mpi_options variable.
				UpperCAmelCase__  :					Optional[int]        =     os.getenv("""SM_FRAMEWORK_PARAMS""" ,      """{}"""					)
				try:
								# Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
								UpperCAmelCase__  :					List[str]        =     json.loads(UpperCamelCase__					)
								if not mpi_options.get("""sagemaker_mpi_enabled""" ,      UpperCamelCase__					):
												return False
				except json.JSONDecodeError:
								return False
				# Lastly, check if the `smdistributed` module is present.
				return importlib.util.find_spec("""smdistributed"""					) is not None


if is_sagemaker_model_parallel_available():
		import smdistributed.modelparallel.torch as smp

		smp.init()


@dataclass
class 		_snake_case					(     a__	):
					lowerCAmelCase      :str   =					field(
					    default=''''''       ,   metadata={'''help''': '''Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer'''}       ,   )




					def 				snake_case__						(      self):
									super().__post_init__()
									warnings.warn(
									    """`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use """
									    """`TrainingArguments` instead."""							,							_lowerCamelCase							,							)




					@cached_property
					def 				snake_case__						(      self):
									logger.info("""PyTorch: setting up devices""")
									if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
													logger.warning(
													    """torch.distributed process group is initialized, but local_rank == -1. """
													    """In order to use Torch DDP, launch your script with `python -m torch.distributed.launch""")
									if self.no_cuda:
													UpperCAmelCase__  :					List[Any]        =     torch.device("""cpu""")
													UpperCAmelCase__  :					Optional[int]        =     0
									elif is_sagemaker_model_parallel_available():
													UpperCAmelCase__  :					str        =     smp.local_rank()
													UpperCAmelCase__  :					Any        =     torch.device("""cuda"""							,							_lowerCamelCase)
													UpperCAmelCase__  :					Dict        =     1
									elif is_sagemaker_dp_enabled():
													import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

													torch.distributed.init_process_group(backend="""smddp"""							,							timeout=self.ddp_timeout_delta)
													UpperCAmelCase__  :					Union[str, Any]        =     int(os.getenv("""SMDATAPARALLEL_LOCAL_RANK"""))
													UpperCAmelCase__  :					int        =     torch.device("""cuda"""							,							self.local_rank)
													UpperCAmelCase__  :					Dict        =     1
									elif self.local_rank == -1:
													# if n_gpu is > 1 we'll use nn.DataParallel.
													# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
													# Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
													# trigger an error that a device index is missing. Index 0 takes into account the
													# GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
													# will use the first GPU in that env, i.e. GPU#1
													UpperCAmelCase__  :					int        =     torch.device("""cuda:0""" if torch.cuda.is_available() else """cpu""")
													# Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
													# the default value.
													UpperCAmelCase__  :					int        =     torch.cuda.device_count()
									else:
													# Here, we'll use torch.distributed.
													# Initializes the distributed backend which will take care of synchronizing nodes/GPUs
													if not torch.distributed.is_initialized():
																	torch.distributed.init_process_group(backend="""nccl"""							,							timeout=self.ddp_timeout_delta)
													UpperCAmelCase__  :					Dict        =     torch.device("""cuda"""							,							self.local_rank)
													UpperCAmelCase__  :					Tuple        =     1

									if device.type == "cuda":
													torch.cuda.set_device(_lowerCamelCase)

									return device




					@property
					def 				snake_case__						(      self):
									if is_sagemaker_model_parallel_available():
													return smp.dp_size()

									return super().world_size




					@property
					def 				snake_case__						(      self):
									return not is_sagemaker_model_parallel_available()




					@property
					def 				snake_case__						(      self):
									return False



'''simple docstring'''



import importlib
import os
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, Optional, Union

import torch

from ..utils import BaseOutput


__A	='scheduler_config.json'


class 		_snake_case					(     a__	):
					lowerCAmelCase      :Union[str, Any]   =					1
					lowerCAmelCase      :Union[str, Any]   =					2
					lowerCAmelCase      :Optional[int]   =					3
					lowerCAmelCase      :Optional[int]   =					4
					lowerCAmelCase      :Any   =					5
					lowerCAmelCase      :Tuple   =					6
					lowerCAmelCase      :List[Any]   =					7
					lowerCAmelCase      :str   =					8
					lowerCAmelCase      :List[Any]   =					9
					lowerCAmelCase      :List[str]   =					10
					lowerCAmelCase      :Union[str, Any]   =					11
					lowerCAmelCase      :Optional[int]   =					12
					lowerCAmelCase      :str   =					13
					lowerCAmelCase      :Dict   =					14


@dataclass
class 		_snake_case					(     a__	):
					lowerCAmelCase      :torch.FloatTensor


class 		_snake_case					:
					lowerCAmelCase      :str   =					SCHEDULER_CONFIG_NAME
					lowerCAmelCase      :Union[str, Any]   =					[]
					lowerCAmelCase      :List[str]   =					True




					@classmethod
					def 				snake_case__						(      cls							,							_lowerCamelCase = None							,							_lowerCamelCase = None							,							_lowerCamelCase=False							,							**_lowerCamelCase							,							):
								UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__  :					Optional[int]        =     cls.load_config(
								    pretrained_model_name_or_path=_lowerCamelCase							,							subfolder=_lowerCamelCase							,							return_unused_kwargs=_lowerCamelCase							,							return_commit_hash=_lowerCamelCase							,							**_lowerCamelCase							,							)
								return cls.from_config(_lowerCamelCase							,							return_unused_kwargs=_lowerCamelCase							,							**_lowerCamelCase)




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase = False							,							**_lowerCamelCase):
								self.save_config(save_directory=_lowerCamelCase							,							push_to_hub=_lowerCamelCase							,							**_lowerCamelCase)




					@property
					def 				snake_case__						(      self):
								return self._get_compatibles()




					@classmethod
					def 				snake_case__						(      cls):
								UpperCAmelCase__  :					List[Any]        =     list(set([cls.__name__] + cls._compatibles))
								UpperCAmelCase__  :					int        =     importlib.import_module(__name__.split(""".""")[0])
								UpperCAmelCase__  :					Dict        =     [
								    getattr(_lowerCamelCase							,							_lowerCamelCase) for c in compatible_classes_str if hasattr(_lowerCamelCase							,							_lowerCamelCase)
								]
								return compatible_classes