





import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


_lowercase		:  Any			       =logging.get_logger(__name__)


def 		_SCREAMING_SNAKE_CASE     ( ):
		# Get the sagemaker specific mp parameters from smp_options variable.
		lowerCamelCase_	:     Tuple 						=      os.getenv('SM_HP_MP_PARAMETERS'							,'{}'						)
		try:
				# Parse it and check the field "partitions" is included, it is required for model parallel.
				lowerCamelCase_	:     Dict 						=      json.loads(lowerCAmelCase__						)
				if "partitions" not in smp_options:
						return False
		except json.JSONDecodeError:
				return False

		# Get the sagemaker specific framework parameters from mpi_options variable.
		lowerCamelCase_	:     Union[str, Any] 						=      os.getenv('SM_FRAMEWORK_PARAMS'							,'{}'						)
		try:
				# Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
				lowerCamelCase_	:     List[Any] 						=      json.loads(lowerCAmelCase__						)
				if not mpi_options.get('sagemaker_mpi_enabled'							,lowerCAmelCase__						):
						return False
		except json.JSONDecodeError:
				return False
		# Lastly, check if the `smdistributed` module is present.
		return importlib.util.find_spec('smdistributed'						) is not None


if is_sagemaker_model_parallel_available():
		import smdistributed.modelparallel.torch as smp

		smp.init()






@dataclass
class 						UpperCamelCase_  (  __a							):
					_a      :						str						    =					field(
					    default=''  ,     metadata={'help': 'Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer'}  ,     )



					def   __a     (		self     :     Dict						):
							super().__post_init__()
							warnings.warn(
							    '`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use '
							    '`TrainingArguments` instead.'       ,      _lowerCamelCase       ,      )



					@cached_property
					def   __a     (		self     :     List[Any]						):
							logger.info('PyTorch: setting up devices'						)
							if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
									logger.warning(
									    'torch.distributed process group is initialized, but local_rank == -1. '
									    'In order to use Torch DDP, launch your script with `python -m torch.distributed.launch'						)
							if self.no_cuda:
									lowerCamelCase_	:     List[Any] 						=      torch.device('cpu'						)
									lowerCamelCase_	:     str 						=      0
							elif is_sagemaker_model_parallel_available():
									lowerCamelCase_	:     Tuple 						=      smp.local_rank()
									lowerCamelCase_	:     List[str] 						=      torch.device('cuda'       ,      _lowerCamelCase						)
									lowerCamelCase_	:     int 						=      1
							elif is_sagemaker_dp_enabled():
									import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

									torch.distributed.init_process_group(backend='smddp'       ,      timeout=self.ddp_timeout_delta						)
									lowerCamelCase_	:     int 						=      int(os.getenv('SMDATAPARALLEL_LOCAL_RANK'						)						)
									lowerCamelCase_	:     Dict 						=      torch.device('cuda'       ,      self.local_rank						)
									lowerCamelCase_	:     Dict 						=      1
							elif self.local_rank == -1:
									# if n_gpu is > 1 we'll use nn.DataParallel.
									# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
									# Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
									# trigger an error that a device index is missing. Index 0 takes into account the
									# GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
									# will use the first GPU in that env, i.e. GPU#1
									lowerCamelCase_	:     Any 						=      torch.device('cuda:0' if torch.cuda.is_available() else 'cpu'						)
									# Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
									# the default value.
									lowerCamelCase_	:     str 						=      torch.cuda.device_count()
							else:
									# Here, we'll use torch.distributed.
									# Initializes the distributed backend which will take care of synchronizing nodes/GPUs
									if not torch.distributed.is_initialized():
											torch.distributed.init_process_group(backend='nccl'       ,      timeout=self.ddp_timeout_delta						)
									lowerCamelCase_	:     Optional[Any] 						=      torch.device('cuda'       ,      self.local_rank						)
									lowerCamelCase_	:     Dict 						=      1

							if device.type == "cuda":
									torch.cuda.set_device(_lowerCamelCase						)

							return device



					@property
					def   __a     (		self     :     int						):
							if is_sagemaker_model_parallel_available():
									return smp.dp_size()

							return super().world_size



					@property
					def   __a     (		self     :     Tuple						):
							return not is_sagemaker_model_parallel_available()





					@property
					def   __a     (		self     :     Optional[Any]						):
							return False


"""simple docstring"""






from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


_lowercase							=				logging.get_logger(__name__)


_lowercase							=				{
    '''Salesforce/codegen-350M-nl''': '''https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json''',
    '''Salesforce/codegen-350M-multi''': '''https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json''',
    '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json''',
    '''Salesforce/codegen-2B-nl''': '''https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-2B-multi''': '''https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-2B-mono''': '''https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-6B-nl''': '''https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-6B-multi''': '''https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-6B-mono''': '''https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-16B-nl''': '''https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-16B-multi''': '''https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-16B-mono''': '''https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json''',
}

class 					__a     ( __a       ):



							'''simple docstring'''







							_lowerCamelCase					:			List[Any]          =   """codegen"""
							_lowerCamelCase					:			List[str]          =   {
							    """max_position_embeddings""": """n_positions""",
							    """hidden_size""": """n_embd""",
							    """num_attention_heads""": """n_head""",
							    """num_hidden_layers""": """n_layer""",
							}
							def __init__(     self  ,    _lowerCamelCase=50_400  ,    _lowerCamelCase=2_048  ,    _lowerCamelCase=2_048  ,    _lowerCamelCase=4_096  ,    _lowerCamelCase=28  ,    _lowerCamelCase=16  ,    _lowerCamelCase=64  ,    _lowerCamelCase=None  ,    _lowerCamelCase="gelu_new"  ,    _lowerCamelCase=0.0  ,    _lowerCamelCase=0.0  ,    _lowerCamelCase=0.0  ,    _lowerCamelCase=1e-5  ,    _lowerCamelCase=0.02  ,    _lowerCamelCase=True  ,    _lowerCamelCase=50_256  ,    _lowerCamelCase=50_256  ,    _lowerCamelCase=False  ,    **_lowerCamelCase  ,    )			->							Union[str, Any]:




										'''simple docstring'''
										__lowercase       =       vocab_size
										__lowercase       =       n_ctx
										__lowercase       =       n_positions
										__lowercase       =       n_embd
										__lowercase       =       n_layer
										__lowercase       =       n_head
										__lowercase       =       n_inner
										__lowercase       =       rotary_dim
										__lowercase       =       activation_function
										__lowercase       =       resid_pdrop
										__lowercase       =       embd_pdrop
										__lowercase       =       attn_pdrop
										__lowercase       =       layer_norm_epsilon
										__lowercase       =       initializer_range
										__lowercase       =       use_cache

										__lowercase       =       bos_token_id
										__lowercase       =       eos_token_id

										super().__init__(
										    bos_token_id=_lowerCamelCase  ,    eos_token_id=_lowerCamelCase  ,    tie_word_embeddings=_lowerCamelCase  ,    **_lowerCamelCase       )





class 					__a     ( __a       ):



							'''simple docstring'''







							def __init__(     self  ,    _lowerCamelCase  ,    _lowerCamelCase = "default"  ,    _lowerCamelCase = None  ,    _lowerCamelCase = False  ,    )			->							str:




										'''simple docstring'''
										super().__init__(_lowerCamelCase  ,    task=_lowerCamelCase  ,    patching_specs=_lowerCamelCase  ,    use_past=_lowerCamelCase       )
										if not getattr(self._config  ,    "pad_token_id"  ,    _lowerCamelCase       ):
													# TODO: how to do that better?
													__lowercase       =       0
							@property
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Mapping[str, Mapping[int, str]]:




										'''simple docstring'''
										__lowercase       =       OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}       )
										if self.use_past:
													self.fill_with_past_key_values_(_lowerCamelCase  ,    direction="inputs"       )
													__lowercase       =       {0: "batch", 1: "past_sequence + sequence"}
										else:
													__lowercase       =       {0: "batch", 1: "sequence"}

										return common_inputs
							@property
							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										return self._config.n_layer
							@property
							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										return self._config.n_head
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase = -1  ,    _lowerCamelCase = -1  ,    _lowerCamelCase = False  ,    _lowerCamelCase = None  ,    )			->							Mapping[str, Any]:




										'''simple docstring'''
										__lowercase       =       super(_lowerCamelCase  ,    self       ).generate_dummy_inputs(
										    _lowerCamelCase  ,    batch_size=_lowerCamelCase  ,    seq_length=_lowerCamelCase  ,    is_pair=_lowerCamelCase  ,    framework=_lowerCamelCase       )

										# We need to order the input in the way they appears in the forward()
										__lowercase       =       OrderedDict({"input_ids": common_inputs["input_ids"]}       )

										# Need to add the past_keys
										if self.use_past:
													if not is_torch_available():
																raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."       )
													else:
																import torch

																__lowercase   ,					__lowercase       =       common_inputs["input_ids"].shape
																# Not using the same length for past_key_values
																__lowercase       =       seqlen + 2
																__lowercase       =       (
																    batch,
																    self.num_attention_heads,
																    past_key_values_length,
																    self._config.hidden_size // self.num_attention_heads,
																)
																__lowercase       =       [
																    (torch.zeros(_lowerCamelCase       ), torch.zeros(_lowerCamelCase       )) for _ in range(self.num_layers       )
																]

										__lowercase       =       common_inputs["attention_mask"]
										if self.use_past:
													__lowercase       =       ordered_inputs["attention_mask"].dtype
													__lowercase       =       torch.cat(
													    [ordered_inputs["attention_mask"], torch.ones(_lowerCamelCase  ,    _lowerCamelCase  ,    dtype=_lowerCamelCase       )]  ,    dim=1       )

										return ordered_inputs





							@property
							def   SCREAMING_SNAKE_CASE						(     self       )			->							int:




										'''simple docstring'''
										return 13





