



import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


__snake_case     :Optional[Any]			       =logging.get_logger(__name__)






def  lowerCamelCase_	(      )       ->							str:

							'''simple docstring'''






							A      						=    os.getenv('SM_HP_MP_PARAMETERS'			,	'{}'				)
							try:
														# Parse it and check the field "partitions" is included, it is required for model parallel.
														A      						=    json.loads(lowerCAmelCase__				)
														if "partitions" not in smp_options:
																					return False
							except json.JSONDecodeError:
														return False

							# Get the sagemaker specific framework parameters from mpi_options variable.
							A      						=    os.getenv('SM_FRAMEWORK_PARAMS'			,	'{}'				)
							try:
														# Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
														A      						=    json.loads(lowerCAmelCase__				)
														if not mpi_options.get('sagemaker_mpi_enabled'			,	lowerCAmelCase__				):
																					return False
							except json.JSONDecodeError:
														return False
							# Lastly, check if the `smdistributed` module is present.
							return importlib.util.find_spec('smdistributed'				) is not None


if is_sagemaker_model_parallel_available():
				import smdistributed.modelparallel.torch as smp

				smp.init()



@dataclass
class        lowerCAmelCase__					(  _lowerCamelCase       ):
				A_   :     str								=     field(
				    default='' ,				metadata={'help': 'Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer'} ,				)


				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     str:
											super().__post_init__()
											warnings.warn(
											    '`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use '
											    '`TrainingArguments` instead.'						, __UpperCamelCase						, )


				@cached_property
				def __UpperCamelCase						(							self      :  List[str]       )		->     "torch.device":
											logger.info('PyTorch: setting up devices'       )
											if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
																		logger.warning(
																		    'torch.distributed process group is initialized, but local_rank == -1. '
																		    'In order to use Torch DDP, launch your script with `python -m torch.distributed.launch'       )
											if self.no_cuda:
																		A      						=    torch.device('cpu'       )
																		A      						=    0
											elif is_sagemaker_model_parallel_available():
																		A      						=    smp.local_rank()
																		A      						=    torch.device('cuda'						, __UpperCamelCase       )
																		A      						=    1
											elif is_sagemaker_dp_enabled():
																		import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

																		torch.distributed.init_process_group(backend='smddp'						, timeout=self.ddp_timeout_delta       )
																		A      						=    int(os.getenv('SMDATAPARALLEL_LOCAL_RANK'       )       )
																		A      						=    torch.device('cuda'						, self.local_rank       )
																		A      						=    1
											elif self.local_rank == -1:
																		# if n_gpu is > 1 we'll use nn.DataParallel.
																		# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
																		# Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
																		# trigger an error that a device index is missing. Index 0 takes into account the
																		# GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
																		# will use the first GPU in that env, i.e. GPU#1
																		A      						=    torch.device('cuda:0' if torch.cuda.is_available() else 'cpu'       )
																		# Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
																		# the default value.
																		A      						=    torch.cuda.device_count()
											else:
																		# Here, we'll use torch.distributed.
																		# Initializes the distributed backend which will take care of synchronizing nodes/GPUs
																		if not torch.distributed.is_initialized():
																									torch.distributed.init_process_group(backend='nccl'						, timeout=self.ddp_timeout_delta       )
																		A      						=    torch.device('cuda'						, self.local_rank       )
																		A      						=    1

											if device.type == "cuda":
																		torch.cuda.set_device(__UpperCamelCase       )

											return device


				@property
				def __UpperCamelCase						(							self      :  List[str]       )		->     List[Any]:
											if is_sagemaker_model_parallel_available():
																		return smp.dp_size()

											return super().world_size


				@property
				def __UpperCamelCase						(							self      :  Any       )		->     Dict:
											return not is_sagemaker_model_parallel_available()


				@property
				def __UpperCamelCase						(							self      :  int       )		->     Any:
											return False




'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase:   List[Any]					=		{'configuration_reformer': ['REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ReformerConfig']}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCAmelCase:   Dict					=		['ReformerTokenizer']

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCAmelCase:   Tuple					=		['ReformerTokenizerFast']

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCAmelCase:   List[Any]					=		[
         'REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
         'ReformerAttention',
         'ReformerForMaskedLM',
         'ReformerForQuestionAnswering',
         'ReformerForSequenceClassification',
         'ReformerLayer',
         'ReformerModel',
         'ReformerModelWithLMHead',
         'ReformerPreTrainedModel',
     ]


if TYPE_CHECKING:
     from .configuration_reformer import REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, ReformerConfig

     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_reformer import ReformerTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_reformer_fast import ReformerTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_reformer import (
              REFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
              ReformerAttention,
              ReformerForMaskedLM,
              ReformerForQuestionAnswering,
              ReformerForSequenceClassification,
              ReformerLayer,
              ReformerModel,
              ReformerModelWithLMHead,
              ReformerPreTrainedModel,
          )

else:
     import sys

     lowerCAmelCase:   str					=		_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)