





'''simple docstring'''

import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


lowercase__		:     Tuple			=							logging.get_logger(__name__)







def    a__							(							)						->						str:





     """simple docstring"""

     _UpperCamelCase							       =			os.getenv('''SM_HP_MP_PARAMETERS''',   '''{}''' )
     try:
          # Parse it and check the field "partitions" is included, it is required for model parallel.
          _UpperCamelCase							       =			json.loads(lowercase )
          if "partitions" not in smp_options:
               return False
     except json.JSONDecodeError:
          return False

     # Get the sagemaker specific framework parameters from mpi_options variable.
     _UpperCamelCase							       =			os.getenv('''SM_FRAMEWORK_PARAMS''',   '''{}''' )
     try:
          # Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
          _UpperCamelCase							       =			json.loads(lowercase )
          if not mpi_options.get('''sagemaker_mpi_enabled''',   lowercase ):
               return False
     except json.JSONDecodeError:
          return False
     # Lastly, check if the `smdistributed` module is present.
     return importlib.util.find_spec('''smdistributed''' ) is not None


if is_sagemaker_model_parallel_available():
   import smdistributed.modelparallel.torch as smp

   smp.init()




@dataclass
class 							__lowerCAmelCase	(       __magic_name__					):






       """simple docstring"""
       _snake_case      :	str									=       field(
           default=''		,		metadata={'help': 'Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer'}		,		)





       def snake_case__		(				self  :						List[str]	)       ->	Any:

            '''simple docstring'''




            super().__post_init__()
            warnings.warn(
                '''`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use '''
                '''`TrainingArguments` instead.'''			,					lowerCAmelCase__			,					)





       @cached_property
       def snake_case__		(				self  :						Any	)       ->	"torch.device":

            '''simple docstring'''




            logger.info('''PyTorch: setting up devices'''	)
            if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
                 logger.warning(
                     '''torch.distributed process group is initialized, but local_rank == -1. '''
                     '''In order to use Torch DDP, launch your script with `python -m torch.distributed.launch'''	)
            if self.no_cuda:
                 _UpperCamelCase							       =			torch.device('''cpu'''	)
                 _UpperCamelCase							       =			0
            elif is_sagemaker_model_parallel_available():
                 _UpperCamelCase							       =			smp.local_rank()
                 _UpperCamelCase							       =			torch.device('''cuda'''			,					lowerCAmelCase__	)
                 _UpperCamelCase							       =			1
            elif is_sagemaker_dp_enabled():
                 import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

                 torch.distributed.init_process_group(backend='''smddp'''			,					timeout=self.ddp_timeout_delta	)
                 _UpperCamelCase							       =			int(os.getenv('''SMDATAPARALLEL_LOCAL_RANK'''	)	)
                 _UpperCamelCase							       =			torch.device('''cuda'''			,					self.local_rank	)
                 _UpperCamelCase							       =			1
            elif self.local_rank == -1:
                 # if n_gpu is > 1 we'll use nn.DataParallel.
                 # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
                 # Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
                 # trigger an error that a device index is missing. Index 0 takes into account the
                 # GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
                 # will use the first GPU in that env, i.e. GPU#1
                 _UpperCamelCase							       =			torch.device('''cuda:0''' if torch.cuda.is_available() else '''cpu'''	)
                 # Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
                 # the default value.
                 _UpperCamelCase							       =			torch.cuda.device_count()
            else:
                 # Here, we'll use torch.distributed.
                 # Initializes the distributed backend which will take care of synchronizing nodes/GPUs
                 if not torch.distributed.is_initialized():
                      torch.distributed.init_process_group(backend='''nccl'''			,					timeout=self.ddp_timeout_delta	)
                 _UpperCamelCase							       =			torch.device('''cuda'''			,					self.local_rank	)
                 _UpperCamelCase							       =			1

            if device.type == "cuda":
                 torch.cuda.set_device(lowerCAmelCase__	)

            return device





       @property
       def snake_case__		(				self  :						str	)       ->	Tuple:

            '''simple docstring'''




            if is_sagemaker_model_parallel_available():
                 return smp.dp_size()

            return super().world_size





       @property
       def snake_case__		(				self  :						Dict	)       ->	Optional[Any]:

            '''simple docstring'''




            return not is_sagemaker_model_parallel_available()







       @property
       def snake_case__		(				self  :						int	)       ->	List[Any]:

            '''simple docstring'''




            return False





from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase   =	logging.get_logger(__name__)

lowercase   =	{
    '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/config.json''',
}





class    __A(					UpperCAmelCase							):
					SCREAMING_SNAKE_CASE           =     '''gpt_neox_japanese'''

					def __init__(						self       :  Union[str, Any] ,       __UpperCamelCase       :  str=3_2_0_0_0 ,       __UpperCamelCase       :  List[Any]=2_5_6_0 ,       __UpperCamelCase       :  Any=3_2 ,       __UpperCamelCase       :  List[str]=3_2 ,       __UpperCamelCase       :  List[str]=4 ,       __UpperCamelCase       :  Dict="gelu" ,       __UpperCamelCase       :  List[Any]=1.00 ,       __UpperCamelCase       :  Any=1_0_0_0_0 ,       __UpperCamelCase       :  Optional[Any]=2_0_4_8 ,       __UpperCamelCase       :  Tuple=0.02 ,       __UpperCamelCase       :  List[str]=1E-5 ,       __UpperCamelCase       :  str=True ,       __UpperCamelCase       :  str=3_1_9_9_6 ,       __UpperCamelCase       :  int=3_1_9_9_9 ,       __UpperCamelCase       :  Optional[Any]=0.1 ,       __UpperCamelCase       :  Tuple=0.0 ,       **__UpperCamelCase       :  List[str] ,       ):
												super().__init__(bos_token_id=__UpperCamelCase ,       eos_token_id=__UpperCamelCase ,       **__UpperCamelCase )
												lowerCamelCase_					=       vocab_size
												lowerCamelCase_					=       max_position_embeddings
												lowerCamelCase_					=       hidden_size
												lowerCamelCase_					=       num_hidden_layers
												lowerCamelCase_					=       num_attention_heads
												lowerCamelCase_					=       intermediate_multiple_size
												lowerCamelCase_					=       hidden_act
												lowerCamelCase_					=       rotary_pct
												lowerCamelCase_					=       rotary_emb_base
												lowerCamelCase_					=       initializer_range
												lowerCamelCase_					=       layer_norm_eps
												lowerCamelCase_					=       use_cache
												lowerCamelCase_					=       attention_dropout
												lowerCamelCase_					=       hidden_dropout





