




'''simple docstring'''





import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


UpperCamelCase_		:     Tuple          =		logging.get_logger(__name__)





def       __a      (  )		->					int:

     """simple docstring"""




     _snake_case				= os.getenv("SM_HP_MP_PARAMETERS"   ,		"{}"       )
     try:
          # Parse it and check the field "partitions" is included, it is required for model parallel.
          _snake_case				= json.loads(_UpperCamelCase       )
          if "partitions" not in smp_options:
               return False
     except json.JSONDecodeError:
          return False

     # Get the sagemaker specific framework parameters from mpi_options variable.
     _snake_case				= os.getenv("SM_FRAMEWORK_PARAMS"   ,		"{}"       )
     try:
          # Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
          _snake_case				= json.loads(_UpperCamelCase       )
          if not mpi_options.get("sagemaker_mpi_enabled"   ,		_UpperCamelCase       ):
               return False
     except json.JSONDecodeError:
          return False
     # Lastly, check if the `smdistributed` module is present.
     return importlib.util.find_spec("smdistributed"       ) is not None


if is_sagemaker_model_parallel_available():
   import smdistributed.modelparallel.torch as smp

   smp.init()





@dataclass
class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       str       							=		field(
         default=""""""   ,   metadata={"""help""": """Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer"""}   ,   )






     def 							_lowercase      (					self       )				->     str:
          super().__post_init__()
          warnings.warn(
              "`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use "
              "`TrainingArguments` instead."   ,_SCREAMING_SNAKE_CASE   ,)






     @cached_property
     def 							_lowercase      (					self       )				->     "torch.device":
          logger.info("PyTorch: setting up devices"       )
          if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
               logger.warning(
                   "torch.distributed process group is initialized, but local_rank == -1. "
                   "In order to use Torch DDP, launch your script with `python -m torch.distributed.launch"       )
          if self.no_cuda:
               _snake_case				= torch.device("cpu"       )
               _snake_case				= 0
          elif is_sagemaker_model_parallel_available():
               _snake_case				= smp.local_rank()
               _snake_case				= torch.device("cuda"   ,_SCREAMING_SNAKE_CASE       )
               _snake_case				= 1
          elif is_sagemaker_dp_enabled():
               import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

               torch.distributed.init_process_group(backend="smddp"   ,timeout=self.ddp_timeout_delta       )
               _snake_case				= int(os.getenv("SMDATAPARALLEL_LOCAL_RANK"       )       )
               _snake_case				= torch.device("cuda"   ,self.local_rank       )
               _snake_case				= 1
          elif self.local_rank == -1:
               # if n_gpu is > 1 we'll use nn.DataParallel.
               # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
               # Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
               # trigger an error that a device index is missing. Index 0 takes into account the
               # GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
               # will use the first GPU in that env, i.e. GPU#1
               _snake_case				= torch.device("cuda:0" if torch.cuda.is_available() else "cpu"       )
               # Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
               # the default value.
               _snake_case				= torch.cuda.device_count()
          else:
               # Here, we'll use torch.distributed.
               # Initializes the distributed backend which will take care of synchronizing nodes/GPUs
               if not torch.distributed.is_initialized():
                    torch.distributed.init_process_group(backend="nccl"   ,timeout=self.ddp_timeout_delta       )
               _snake_case				= torch.device("cuda"   ,self.local_rank       )
               _snake_case				= 1

          if device.type == "cuda":
               torch.cuda.set_device(_SCREAMING_SNAKE_CASE       )

          return device






     @property
     def 							_lowercase      (					self       )				->     Any:
          if is_sagemaker_model_parallel_available():
               return smp.dp_size()

          return super().world_size






     @property
     def 							_lowercase      (					self       )				->     List[str]:
          return not is_sagemaker_model_parallel_available()






     @property
     def 							_lowercase      (					self       )				->     Dict:
          return False








'''simple docstring'''





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_		:     int          =		logging.get_logger(__name__)

UpperCamelCase_		:     Any          =		{
    '''tiiuae/falcon-40b''': '''https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json''',
    '''tiiuae/falcon-7b''': '''https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json''',
}

class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       List[str]       							=		"""falcon"""
     SCREAMING_SNAKE_CASE_				:       Optional[int]       							=		["""past_key_values"""]






     def __init__(					self   ,_SCREAMING_SNAKE_CASE=65_024   ,_SCREAMING_SNAKE_CASE=4_544   ,_SCREAMING_SNAKE_CASE=32   ,_SCREAMING_SNAKE_CASE=71   ,_SCREAMING_SNAKE_CASE=1e-5   ,_SCREAMING_SNAKE_CASE=0.0_2   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=11   ,_SCREAMING_SNAKE_CASE=11   ,**_SCREAMING_SNAKE_CASE   ,)				->     Dict:
          _snake_case				= vocab_size
          # Backward compatibility with n_embed kwarg
          _snake_case				= kwargs.pop("n_embed"   ,_SCREAMING_SNAKE_CASE       )
          _snake_case				= hidden_size if n_embed is None else n_embed
          _snake_case				= num_hidden_layers
          _snake_case				= num_attention_heads
          _snake_case				= layer_norm_epsilon
          _snake_case				= initializer_range
          _snake_case				= use_cache
          _snake_case				= hidden_dropout
          _snake_case				= attention_dropout

          _snake_case				= bos_token_id
          _snake_case				= eos_token_id
          _snake_case				= num_attention_heads if num_kv_heads is None else num_kv_heads
          _snake_case				= alibi
          _snake_case				= new_decoder_architecture
          _snake_case				= multi_query  # Ignored when new_decoder_architecture is True
          _snake_case				= parallel_attn
          _snake_case				= bias

          super().__init__(bos_token_id=_SCREAMING_SNAKE_CASE   ,eos_token_id=_SCREAMING_SNAKE_CASE   ,**_SCREAMING_SNAKE_CASE       )






     @property
     def 							_lowercase      (					self       )				->     Union[str, Any]:
          return self.hidden_size // self.num_attention_heads






     @property
     def 							_lowercase      (					self       )				->     Optional[int]:
          return not self.alibi


