



"""simple docstring"""

import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment





def    __magic_name__     ( _lowerCamelCase					:			Union[str, Any]	,							_lowerCamelCase					:			Optional[Any]=()	,							_lowerCamelCase					:			Optional[int]=None	,							_lowerCamelCase					:			Dict="no"	,							_lowerCamelCase					:			int="29500"       ):
							__a		:		List[str]						=							False
							__a		:		str						=							False
							if any(key.startswith("""KAGGLE"""       ) for key in os.environ.keys()       ):
														__a		:		Optional[int]						=							True
							elif "IPython" in sys.modules:
														__a		:		int						=							"google.colab" in str(sys.modules["""IPython"""].get_ipython()       )

							try:
														__a		:		Dict						=							PrecisionType(mixed_precision.lower()       )
							except ValueError:
														raise ValueError(
														    F'''Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.'''       )

							if (in_colab or in_kaggle) and (os.environ.get("""TPU_NAME"""	,							__UpperCamelCase       ) is not None):
														# TPU launch
														import torch_xla.distributed.xla_multiprocessing as xmp

														if len(AcceleratorState._shared_state       ) > 0:
																					raise ValueError(
																					    """To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside """
																					    """your training function. Restart your notebook and make sure no cells initializes an """
																					    """`Accelerator`."""       )
														if num_processes is None:
																					__a		:		str						=							8

														__a		:		List[str]						=							PrepareForLaunch(__UpperCamelCase	,							distributed_type="""TPU"""       )
														print(F'''Launching a training on {num_processes} TPU cores.'''       )
														xmp.spawn(__UpperCamelCase	,							args=__UpperCamelCase	,							nprocs=__UpperCamelCase	,							start_method="""fork"""       )
							elif in_colab:
														# No need for a distributed launch otherwise as it's either CPU or one GPU.
														if torch.cuda.is_available():
																					print("""Launching training on one GPU."""       )
														else:
																					print("""Launching training on one CPU."""       )
														function(*__UpperCamelCase       )
							else:
														if num_processes is None:
																					raise ValueError(
																					    """You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call."""       )

														if num_processes > 1:
																					# Multi-GPU launch
																					from torch.multiprocessing import start_processes
																					from torch.multiprocessing.spawn import ProcessRaisedException

																					if len(AcceleratorState._shared_state       ) > 0:
																												raise ValueError(
																												    """To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized """
																												    """inside your training function. Restart your notebook and make sure no cells initializes an """
																												    """`Accelerator`."""       )

																					if torch.cuda.is_initialized():
																												raise ValueError(
																												    """To launch a multi-GPU training from your notebook, you need to avoid running any instruction """
																												    """using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA """
																												    """function."""       )

																					# torch.distributed will expect a few environment variable to be here. We set the ones common to each
																					# process here (the other ones will be set be the launcher).
																					with patch_environment(
																					    world_size=__UpperCamelCase	,							master_addr="""127.0.01"""	,							master_port=__UpperCamelCase	,							mixed_precision=__UpperCamelCase       ):
																												__a		:		Union[str, Any]						=							PrepareForLaunch(__UpperCamelCase	,							distributed_type="""MULTI_GPU"""       )
																												print(F'''Launching training on {num_processes} GPUs.'''       )
																												try:
																																			start_processes(__UpperCamelCase	,							args=__UpperCamelCase	,							nprocs=__UpperCamelCase	,							start_method="""fork"""       )
																												except ProcessRaisedException as e:
																																			if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
																																										raise RuntimeError(
																																										    """CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. """
																																										    """This likely stems from an outside import causing issues once the `notebook_launcher()` is called. """
																																										    """Please review your imports and test them when running the `notebook_launcher()` to identify """
																																										    """which one is problematic."""       ) from e

														else:
																					# No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
																					if is_mps_available():
																												__a		:		Any						=							"1"
																												print("""Launching training on MPS."""       )
																					elif torch.cuda.is_available():
																												print("""Launching training on one GPU."""       )
																					else:
																												print("""Launching training on CPU."""       )
																					function(*__UpperCamelCase       )







def    __magic_name__     ( _lowerCamelCase					:			Any	,							_lowerCamelCase					:			List[Any]=()	,							_lowerCamelCase					:			Optional[Any]=2       ):
							from torch.multiprocessing import start_processes

							with tempfile.NamedTemporaryFile() as tmp_file:
														# torch.distributed will expect a few environment variable to be here. We set the ones common to each
														# process here (the other ones will be set be the launcher).
														with patch_environment(
														    world_size=__UpperCamelCase	,							master_addr="""127.0.01"""	,							master_port="""29500"""	,							accelerate_mixed_precision="""no"""	,							accelerate_debug_rdv_file=tmp_file.name	,							accelerate_use_cpu="""yes"""	,							):
																					__a		:		str						=							PrepareForLaunch(__UpperCamelCase	,							debug=__UpperCamelCase       )
																					start_processes(__UpperCamelCase	,							args=__UpperCamelCase	,							nprocs=__UpperCamelCase	,							start_method="""fork"""       )






from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowerCamelCase		       =      logging.get_logger(__name__)


lowerCamelCase		       =      {
    """Salesforce/codegen-350M-nl""": """https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json""",
    """Salesforce/codegen-350M-multi""": """https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json""",
    """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json""",
    """Salesforce/codegen-2B-nl""": """https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json""",
    """Salesforce/codegen-2B-multi""": """https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json""",
    """Salesforce/codegen-2B-mono""": """https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json""",
    """Salesforce/codegen-6B-nl""": """https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json""",
    """Salesforce/codegen-6B-multi""": """https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json""",
    """Salesforce/codegen-6B-mono""": """https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json""",
    """Salesforce/codegen-16B-nl""": """https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json""",
    """Salesforce/codegen-16B-multi""": """https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json""",
    """Salesforce/codegen-16B-mono""": """https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json""",
}


class 				_a						(       SCREAMING_SNAKE_CASE			):





							'''simple docstring'''

							A				:str =       "codegen"
							A				:Tuple =       {
							    "max_position_embeddings": "n_positions",
							    "hidden_size": "n_embd",
							    "num_attention_heads": "n_head",
							    "num_hidden_layers": "n_layer",
							}



							def __init__(							self ,					__UpperCAmelCase=5_0400 ,					__UpperCAmelCase=2048 ,					__UpperCAmelCase=2048 ,					__UpperCAmelCase=4096 ,					__UpperCAmelCase=28 ,					__UpperCAmelCase=16 ,					__UpperCAmelCase=64 ,					__UpperCAmelCase=None ,					__UpperCAmelCase="gelu_new" ,					__UpperCAmelCase=0.0 ,					__UpperCAmelCase=0.0 ,					__UpperCAmelCase=0.0 ,					__UpperCAmelCase=1E-5 ,					__UpperCAmelCase=0.0_2 ,					__UpperCAmelCase=True ,					__UpperCAmelCase=5_0256 ,					__UpperCAmelCase=5_0256 ,					__UpperCAmelCase=False ,					**__UpperCAmelCase ,					):







											"""simple docstring"""



											a__	:							Dict      						= vocab_size
											a__	:							int      						= n_ctx
											a__	:							str      						= n_positions
											a__	:							Union[str, Any]      						= n_embd
											a__	:							Union[str, Any]      						= n_layer
											a__	:							Optional[int]      						= n_head
											a__	:							Dict      						= n_inner
											a__	:							Optional[Any]      						= rotary_dim
											a__	:							str      						= activation_function
											a__	:							List[Any]      						= resid_pdrop
											a__	:							str      						= embd_pdrop
											a__	:							List[str]      						= attn_pdrop
											a__	:							str      						= layer_norm_epsilon
											a__	:							Any      						= initializer_range
											a__	:							List[str]      						= use_cache

											a__	:							Tuple      						= bos_token_id
											a__	:							List[Any]      						= eos_token_id

											super().__init__(
											    bos_token_id=__UpperCAmelCase ,					eos_token_id=__UpperCAmelCase ,					tie_word_embeddings=__UpperCAmelCase ,					**__UpperCAmelCase				)





class 				_a						(       SCREAMING_SNAKE_CASE			):





							'''simple docstring'''

							def __init__(							self ,					__UpperCAmelCase ,					__UpperCAmelCase = "default" ,					__UpperCAmelCase = None ,					__UpperCAmelCase = False ,					):







											"""simple docstring"""



											super().__init__(__UpperCAmelCase ,					task=__UpperCAmelCase ,					patching_specs=__UpperCAmelCase ,					use_past=__UpperCAmelCase				)
											if not getattr(self._config ,					"pad_token_id" ,					__UpperCAmelCase				):
															# TODO: how to do that better?
															a__	:							Union[str, Any]      						= 0



							@property
							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Optional[Any]      						= OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}				)
											if self.use_past:
															self.fill_with_past_key_values_(__UpperCAmelCase ,					direction="inputs"				)
															a__	:							Any      						= {0: "batch", 1: "past_sequence + sequence"}
											else:
															a__	:							List[Any]      						= {0: "batch", 1: "sequence"}

											return common_inputs



							@property
							def 		_A	(							self				):







											"""simple docstring"""



											return self._config.n_layer



							@property
							def 		_A	(							self				):







											"""simple docstring"""



											return self._config.n_head



							def 		_A	(							self ,					__UpperCAmelCase ,					__UpperCAmelCase = -1 ,					__UpperCAmelCase = -1 ,					__UpperCAmelCase = False ,					__UpperCAmelCase = None ,					):







											"""simple docstring"""



											a__	:							Dict      						= super(__UpperCAmelCase ,					self				).generate_dummy_inputs(
											    __UpperCAmelCase ,					batch_size=__UpperCAmelCase ,					seq_length=__UpperCAmelCase ,					is_pair=__UpperCAmelCase ,					framework=__UpperCAmelCase				)

											# We need to order the input in the way they appears in the forward()
											a__	:							Union[str, Any]      						= OrderedDict({"input_ids": common_inputs["input_ids"]}				)

											# Need to add the past_keys
											if self.use_past:
															if not is_torch_available():
																			raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."				)
															else:
																			import torch

																			a__						,   a__	:							Union[str, Any]      						= common_inputs["input_ids"].shape
																			# Not using the same length for past_key_values
																			a__	:							Any      						= seqlen + 2
																			a__	:							str      						= (
																			    batch,
																			    self.num_attention_heads,
																			    past_key_values_length,
																			    self._config.hidden_size // self.num_attention_heads,
																			)
																			a__	:							str      						= [
																			    (torch.zeros(__UpperCAmelCase				), torch.zeros(__UpperCAmelCase				)) for _ in range(self.num_layers				)
																			]

											a__	:							Union[str, Any]      						= common_inputs["attention_mask"]
											if self.use_past:
															a__	:							str      						= ordered_inputs["attention_mask"].dtype
															a__	:							Union[str, Any]      						= torch.cat(
															    [ordered_inputs["attention_mask"], torch.ones(__UpperCAmelCase ,					__UpperCAmelCase ,					dtype=__UpperCAmelCase				)] ,					dim=1				)

											return ordered_inputs




							@property
							def 		_A	(							self				):







											"""simple docstring"""



											return 13


