
from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast
from ...utils import logging


lowercase						:      Union[str, Any]									= logging.get_logger(__name__)

lowercase						:      Union[str, Any]									= {
    'EleutherAI/gpt-neo-1.3B': 'https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/config.json',
    # See all GPTNeo models at https://huggingface.co/models?filter=gpt_neo
}




class 				lowerCamelCase__							(       __lowercase):




	'''simple docstring'''




	_A			=  'gpt_neo'
	_A			=  ['past_key_values']
	_A			=  {'num_attention_heads': 'num_heads', 'num_hidden_layers': 'num_layers'}







	def __init__(      self	:str  ,     a	:Any=5_0_2_5_7  ,     a	:Union[str, Any]=2_0_4_8  ,     a	:Optional[Any]=2_0_4_8  ,     a	:Dict=2_4  ,     a	:Dict=[[["global", "local"], 1_2]]  ,     a	:List[str]=1_6  ,     a	:Optional[Any]=None  ,     a	:Optional[Any]=2_5_6  ,     a	:List[Any]="gelu_new"  ,     a	:str=0.0  ,     a	:Optional[Any]=0.0  ,     a	:Any=0.0  ,     a	:List[str]=0.1  ,     a	:Optional[Any]=1E-5  ,     a	:List[str]=0.02  ,     a	:Optional[int]=True  ,     a	:Dict=5_0_2_5_6  ,     a	:str=5_0_2_5_6  ,     **a	:List[Any]  ,     )	->				Optional[Any]:
								__UpperCamelCase			:     Optional[int]         =      vocab_size
								__UpperCamelCase			:     Tuple         =      max_position_embeddings
								__UpperCamelCase			:     str         =      hidden_size
								__UpperCamelCase			:     Optional[Any]         =      num_layers
								__UpperCamelCase			:     Union[str, Any]         =      num_heads
								__UpperCamelCase			:     str         =      intermediate_size
								__UpperCamelCase			:     Any         =      window_size
								__UpperCamelCase			:     Union[str, Any]         =      activation_function
								__UpperCamelCase			:     int         =      resid_dropout
								__UpperCamelCase			:     Optional[int]         =      embed_dropout
								__UpperCamelCase			:     List[str]         =      attention_dropout
								__UpperCamelCase			:     Dict         =      classifier_dropout
								__UpperCamelCase			:     Optional[Any]         =      layer_norm_epsilon
								__UpperCamelCase			:     List[Any]         =      initializer_range
								__UpperCamelCase			:     Any         =      use_cache

								__UpperCamelCase			:     int         =      bos_token_id
								__UpperCamelCase			:     Any         =      eos_token_id

								__UpperCamelCase			:     List[str]         =      attention_types
								__UpperCamelCase			:     Tuple         =      self.expand_attention_types_params(a		)

								if len(self.attention_layers		) != self.num_layers:
															raise ValueError(
															    "Configuration for convolutional module is incorrect. "
															    "It is required that `len(config.attention_layers)` == `config.num_layers` "
															    f'but is `len(config.attention_layers) = {len(self.attention_layers		)}`, '
															    f'`config.num_layers = {self.num_layers}`. '
															    "`config.attention_layers` is prepared using `config.attention_types`. "
															    "Please verify the value of `config.attention_types` argument."		)

								super().__init__(bos_token_id=a  ,     eos_token_id=a  ,     **a		)







	@staticmethod
	def      _lowerCamelCase       (      a	:List[Any]		)	->				Any:
								__UpperCamelCase			:     str         =      []
								for item in attention_types:
															for _ in range(item[1]		):
																						attentions.extend(item[0]		)
								return attentions




def 			_SCREAMING_SNAKE_CASE      (	_lowerCamelCase				:    Optional[int]		,						_lowerCamelCase				:    List[str]		,						_lowerCamelCase				:    Optional[int]		,						_lowerCamelCase				:    Any)      ->					int:






							'''simple docstring'''





							import torch

							__UpperCamelCase			:     List[str]         =      input.size()
							__UpperCamelCase			:     int         =      len(_lowerCamelCase)
							__UpperCamelCase			:     Dict         =      shape[dimension]

							__UpperCamelCase			:     Union[str, Any]         =      torch.arange(0		,						_lowerCamelCase		,						_lowerCamelCase)
							__UpperCamelCase			:     List[str]         =      torch.div(sizedim - size		,						_lowerCamelCase		,						rounding_mode="floor") + 1
							__UpperCamelCase			:     Dict         =      torch.arange(_lowerCamelCase) + low_indices[:min_length][:, None]

							__UpperCamelCase			:     Dict         =      [slice(_lowerCamelCase)] * rank
							__UpperCamelCase			:     str         =      indices
							__UpperCamelCase			:     Tuple         =      input[s]

							__UpperCamelCase			:     Tuple         =      list(range(0		,						rank + 1))
							perm.append(perm.pop(dimension + 1))

							return sliced.permute(_lowerCamelCase)




def 			_SCREAMING_SNAKE_CASE      (	_lowerCamelCase				:    Tuple		,						_lowerCamelCase				:    Optional[Any])      ->					Optional[Any]:






							'''simple docstring'''





							import torch

							__UpperCamelCase			:     List[Any]         =      torch.arange(1		,						_lowerCamelCase)
							__UpperCamelCase			:     Optional[int]         =      torch.remainder(_lowerCamelCase		,						_lowerCamelCase)
							__UpperCamelCase			:     Union[str, Any]         =      remainders == 0
							__UpperCamelCase			:     Optional[Any]         =      candidates[divisor_indices]
							__UpperCamelCase			:     str         =      torch.max(_lowerCamelCase)
							return largest_divisor, torch.div(_lowerCamelCase		,						_lowerCamelCase		,						rounding_mode="floor")




class 				lowerCamelCase__							(       __lowercase):




	'''simple docstring'''




	@property
	def      _lowerCamelCase       (      self	:int		)	->				Mapping[str, Mapping[int, str]]:
								__UpperCamelCase			:     Optional[Any]         =      OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}		)
								if self.use_past:
															self.fill_with_past_key_values_(a  ,     direction="inputs"		)
															__UpperCamelCase			:     str         =      {0: "batch", 1: "past_sequence + sequence"}
								else:
															__UpperCamelCase			:     Any         =      {0: "batch", 1: "sequence"}

								return common_inputs







	@property
	def      _lowerCamelCase       (      self	:Any		)	->				int:
								return self._config.num_heads







	def      _lowerCamelCase       (      self	:List[Any]  ,     a	:PreTrainedTokenizer  ,     a	:int = -1  ,     a	:int = -1  ,     a	:bool = False  ,     a	:Optional[TensorType] = None  ,     )	->				Mapping[str, Any]:
								__UpperCamelCase			:     str         =      super(a  ,     self		).generate_dummy_inputs(
								    a  ,     batch_size=a  ,     seq_length=a  ,     is_pair=a  ,     framework=a		)

								# We need to order the input in the way they appears in the forward()
								__UpperCamelCase			:     Optional[Any]         =      OrderedDict({"input_ids": common_inputs["input_ids"]}		)

								# Need to add the past_keys
								if self.use_past:
															if not is_torch_available():
																						raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."		)
															else:
																						import torch

																						__UpperCamelCase			:     List[str]         =      common_inputs["input_ids"].shape
																						# Not using the same length for past_key_values
																						__UpperCamelCase			:     List[str]         =      seqlen + 2
																						__UpperCamelCase			:     Optional[int]         =      (
																						    batch,
																						    self.num_attention_heads,
																						    past_key_values_length,
																						    self._config.hidden_size // self.num_attention_heads,
																						)
																						__UpperCamelCase			:     Tuple         =      [
																						    (torch.zeros(a		), torch.zeros(a		)) for _ in range(self.num_layers		)
																						]

								__UpperCamelCase			:     Dict         =      common_inputs["attention_mask"]
								if self.use_past:
															__UpperCamelCase			:     Optional[int]         =      ordered_inputs["attention_mask"].dtype
															__UpperCamelCase			:     Optional[Any]         =      torch.cat(
															    [ordered_inputs["attention_mask"], torch.ones(a  ,     a  ,     dtype=a		)]  ,     dim=1		)

								return ordered_inputs







	@property
	def      _lowerCamelCase       (      self	:Union[str, Any]		)	->				int:
								return 1_3

from .constants import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SAFE_WEIGHTS_INDEX_NAME,
    SAFE_WEIGHTS_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    TORCH_LAUNCH_PARAMS,
    WEIGHTS_INDEX_NAME,
    WEIGHTS_NAME,
)
from .dataclasses import (
    BnbQuantizationConfig,
    ComputeEnvironment,
    CustomDtype,
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    DynamoBackend,
    FPaRecipeKwargs,
    FullyShardedDataParallelPlugin,
    GradientAccumulationPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    KwargsHandler,
    LoggerType,
    MegatronLMPlugin,
    PrecisionType,
    ProjectConfiguration,
    RNGType,
    SageMakerDistributedType,
    TensorInformation,
    TorchDynamoPlugin,
)
from .environment import get_int_from_env, parse_choice_from_env, parse_flag_from_env
from .imports import (
    get_ccl_version,
    is_abit_bnb_available,
    is_abit_bnb_available,
    is_aim_available,
    is_bfaa_available,
    is_bnb_available,
    is_botoa_available,
    is_ccl_available,
    is_comet_ml_available,
    is_datasets_available,
    is_deepspeed_available,
    is_fpa_available,
    is_ipex_available,
    is_megatron_lm_available,
    is_mlflow_available,
    is_mps_available,
    is_npu_available,
    is_rich_available,
    is_safetensors_available,
    is_sagemaker_available,
    is_tensorboard_available,
    is_tpu_available,
    is_transformers_available,
    is_wandb_available,
    is_xpu_available,
)
from .modeling import (
    check_device_map,
    check_tied_parameters_in_config,
    check_tied_parameters_on_same_device,
    compute_module_sizes,
    convert_file_size_to_int,
    dtype_byte_size,
    find_tied_parameters,
    get_balanced_memory,
    get_max_layer_size,
    get_max_memory,
    get_mixed_precision_context_manager,
    id_tensor_storage,
    infer_auto_device_map,
    load_checkpoint_in_model,
    load_offloaded_weights,
    load_state_dict,
    named_module_tensors,
    retie_parameters,
    set_module_tensor_to_device,
    shard_checkpoint,
)
from .offload import (
    OffloadedWeightsLoader,
    PrefixedDataset,
    extract_submodules_state_dict,
    load_offloaded_weight,
    offload_state_dict,
    offload_weight,
    save_offload_index,
)
from .operations import (
    broadcast,
    broadcast_object_list,
    concatenate,
    convert_outputs_to_fpaa,
    convert_to_fpaa,
    find_batch_size,
    find_device,
    gather,
    gather_object,
    get_data_structure,
    honor_type,
    initialize_tensors,
    is_namedtuple,
    is_tensor_information,
    is_torch_tensor,
    listify,
    pad_across_processes,
    recursively_apply,
    reduce,
    send_to_device,
    slice_tensors,
)
from .versions import compare_versions, is_torch_version


if is_deepspeed_available():
		from .deepspeed import (
		    DeepSpeedEngineWrapper,
		    DeepSpeedOptimizerWrapper,
		    DeepSpeedSchedulerWrapper,
		    DummyOptim,
		    DummyScheduler,
		    HfDeepSpeedConfig,
		)

from .bnb import has_abit_bnb_layers, load_and_quantize_model
from .fsdp_utils import load_fsdp_model, load_fsdp_optimizer, save_fsdp_model, save_fsdp_optimizer
from .launch import (
    PrepareForLaunch,
    _filter_args,
    prepare_deepspeed_cmd_env,
    prepare_multi_gpu_env,
    prepare_sagemager_args_inputs,
    prepare_simple_launcher_cmd_env,
    prepare_tpu,
)
from .megatron_lm import (
    AbstractTrainStep,
    BertTrainStep,
    GPTTrainStep,
    MegatronEngine,
    MegatronLMDummyDataLoader,
    MegatronLMDummyScheduler,
    MegatronLMOptimizerWrapper,
    MegatronLMSchedulerWrapper,
    TaTrainStep,
    avg_losses_across_data_parallel_group,
    gather_across_data_parallel_groups,
)
from .megatron_lm import initialize as megatron_lm_initialize
from .megatron_lm import prepare_data_loader as megatron_lm_prepare_data_loader
from .megatron_lm import prepare_model as megatron_lm_prepare_model
from .megatron_lm import prepare_optimizer as megatron_lm_prepare_optimizer
from .megatron_lm import prepare_scheduler as megatron_lm_prepare_scheduler
from .memory import find_executable_batch_size, release_memory
from .other import (
    extract_model_from_parallel,
    get_pretty_name,
    is_port_in_use,
    merge_dicts,
    patch_environment,
    save,
    wait_for_everyone,
    write_basic_config,
)
from .random import set_seed, synchronize_rng_state, synchronize_rng_states
from .torch_xla import install_xla
from .tqdm import tqdm
from .transformer_engine import convert_model, has_transformer_engine_layers