import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
							import torch

_lowerCamelCase    :      str	 =						logging.get_logger(__name__)





def 		__lowerCamelCase							():
					return importlib.util.find_spec("deepspeed"     ) is not None


if is_accelerate_available() and is_deepspeed_available():
							from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
							# Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
							# Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
							from builtins import object as DeepSpeedConfig


class   lowercase			(   __A   ):




							def __init__(  self      :    Any		,							_UpperCamelCase      :    Any					)    -> Tuple:



												'''simple docstring'''

												set_hf_deepspeed_config(self					)
												dep_version_check("accelerate"					)
												dep_version_check("deepspeed"					)
												super().__init__(lowerCamelCase__					)



class   lowercase			(   __A   ):




							def __init__(  self      :    Any		,							_UpperCamelCase      :    List[str]					)    -> Optional[int]:



												'''simple docstring'''

												super().__init__(lowerCamelCase__					)
												SCREAMING_SNAKE_CASE				=				None
												SCREAMING_SNAKE_CASE				=				[]




							def 		__snake_case(  self      :    Optional[Any]					)    -> List[str]:



												'''simple docstring'''

												if self._dtype is None:
																	raise ValueError("trainer_config_process() wasn't called yet to tell dtype"					)
												return self._dtype




							def 		__snake_case(  self      :    Optional[Any]		,							_UpperCamelCase      :    List[Any]					)    -> Optional[int]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE				=				self.get_value(lowerCamelCase__					)
												if val is None:
																	return False
												else:
																	return val == "auto"




							def 		__snake_case(  self      :    int		,							_UpperCamelCase      :    Optional[int]		,							_UpperCamelCase      :    Any		,							_UpperCamelCase      :    Any=None		,							_UpperCamelCase      :    str=True					)    -> Any:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE	,      SCREAMING_SNAKE_CASE				=				self.find_config_node(lowerCamelCase__					)
												if config is None:
																	return

												if config.get(lowerCamelCase__					) == "auto":
																	SCREAMING_SNAKE_CASE				=				hf_val
																	return

												if not must_match:
																	return

												SCREAMING_SNAKE_CASE				=				config.get(lowerCamelCase__					)
												if ds_val is not None and ds_val != hf_val:
																	self.mismatches.append(F"- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}"					)

							lowercase__  :				Union[str, Any]									=							partialmethod(__A ,				must_match=__A   )




							def 		__snake_case(  self      :    Any		,							_UpperCamelCase      :    str					)    -> Optional[int]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE				=				args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
												self.fill_match(
												    "train_micro_batch_size_per_gpu"		,							args.per_device_train_batch_size		,							"per_device_train_batch_size"					)
												self.fill_match("gradient_accumulation_steps"		,							args.gradient_accumulation_steps		,							"gradient_accumulation_steps"					)
												self.fill_match("train_batch_size"		,							lowerCamelCase__		,							"train_batch_size (calculated)"					)
												self.fill_match("gradient_clipping"		,							args.max_grad_norm		,							"max_grad_norm"					)

												self.fill_match("optimizer.params.lr"		,							args.learning_rate		,							"learning_rate"					)
												self.fill_match("optimizer.params.betas"		,							[args.adam_betaa, args.adam_betaa]		,							"adam_beta1+adam_beta2"					)
												self.fill_match("optimizer.params.eps"		,							args.adam_epsilon		,							"adam_epsilon"					)
												self.fill_match("optimizer.params.weight_decay"		,							args.weight_decay		,							"weight_decay"					)

												self.fill_only("scheduler.params.warmup_min_lr"		,							0					)  # not a trainer arg
												self.fill_match("scheduler.params.warmup_max_lr"		,							args.learning_rate		,							"learning_rate"					)
												# total_num_steps - will get set in trainer_config_finalize

												# fp16
												if args.fpaa or args.fpaa_full_eval:
																	SCREAMING_SNAKE_CASE				=				"apex" if args.fpaa_backend == "apex" else "amp"
												else:
																	SCREAMING_SNAKE_CASE				=				None

												if args.save_on_each_node:
																	# deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
																	SCREAMING_SNAKE_CASE				=				self.config.get("checkpoint"		,							{}					)
																	SCREAMING_SNAKE_CASE				=				args.save_on_each_node

												# amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
												# any here unless the user did the work
												self.fill_match(
												    "fp16.enabled"		,							((args.fpaa or args.fpaa_full_eval) and fpaa_backend == "amp")		,							"fp16|fp16_full_eval+fp16_backend(amp)"		,							)

												# apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
												# ZeRO features
												self.fill_match("amp.enabled"		,							fpaa_backend == "apex"		,							"fp16+fp16_backend(apex)"					)
												self.fill_match("amp.opt_level"		,							args.fpaa_opt_level		,							"fp16_opt_level"					)

												self.fill_match("bf16.enabled"		,							(args.bfaa or args.bfaa_full_eval)		,							"bf16|bf16_full_eval"					)

												# deepspeed's default mode is fp16 unless there is a config that says differently
												if self.is_true("bf16.enabled"					):
																	SCREAMING_SNAKE_CASE				=				torch.bfloataa
												elif self.is_false("fp16.enabled"					):
																	SCREAMING_SNAKE_CASE				=				torch.floataa
												else:
																	SCREAMING_SNAKE_CASE				=				torch.floataa





							def 		__snake_case(  self      :    List[Any]		,							_UpperCamelCase      :    str		,							_UpperCamelCase      :    Any		,							_UpperCamelCase      :    Union[str, Any]					)    -> List[str]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE				=				[
												    "zero_optimization.reduce_bucket_size",
												    "zero_optimization.stage3_prefetch_bucket_size",
												    "zero_optimization.stage3_param_persistence_threshold",
												]
												SCREAMING_SNAKE_CASE				=				[x for x in hidden_size_based_keys if self.is_auto(lowerCamelCase__					)]

												if len(lowerCamelCase__					) > 0:
																	if hasattr(model.config		,							"hidden_size"					):
																						SCREAMING_SNAKE_CASE				=				model.config.hidden_size
																	elif hasattr(model.config		,							"hidden_sizes"					):
																						# if there are many hidden sizes pick the largest one
																						SCREAMING_SNAKE_CASE				=				max(model.config.hidden_sizes					)
																	else:
																						raise ValueError(
																						    "The model's config file has neither `hidden_size` nor `hidden_sizes` entry, "
																						    "therefore it's not possible to automatically fill out the following `auto` entries "
																						    F"in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing "
																						    "`auto` values for these keys with an integer value of your choice."					)

																	self.fill_only("zero_optimization.reduce_bucket_size"		,							hidden_size * hidden_size					)
																	if self.is_zeroa():
																						# automatically assign the optimal config values based on model config
																						self.fill_only("zero_optimization.stage3_prefetch_bucket_size"		,							0.9 * hidden_size * hidden_size					)
																						self.fill_only("zero_optimization.stage3_param_persistence_threshold"		,							10 * hidden_size					)

        # scheduler
												self.fill_match("scheduler.params.total_num_steps"		,							lowerCamelCase__		,							"num_training_steps (calculated)"					)
												self.fill_match("scheduler.params.warmup_num_steps"		,							args.get_warmup_steps(lowerCamelCase__					)		,							"warmup_steps"					)

												if len(self.mismatches					) > 0:
																	SCREAMING_SNAKE_CASE				=				"\n".join(self.mismatches					)
																	raise ValueError(
																	    "Please correct the following DeepSpeed config values that mismatch TrainingArguments"
																	    F" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'."					)



_lowerCamelCase    :      Tuple	 =						None





def 		__lowerCamelCase							(UpperCAmelCase__   :							Tuple     ):
					global _hf_deepspeed_config_weak_ref
					# will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
					SCREAMING_SNAKE_CASE				=				weakref.ref(a__     )





def 		__lowerCamelCase							():
					global _hf_deepspeed_config_weak_ref
					SCREAMING_SNAKE_CASE				=				None





def 		__lowerCamelCase							():




					if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
										return _hf_deepspeed_config_weak_ref().is_zeroa()
					else:
										return False





def 		__lowerCamelCase							():




					if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
										return _hf_deepspeed_config_weak_ref().config
					else:
										return None





def 		__lowerCamelCase							(UpperCAmelCase__   :							Tuple					,       UpperCAmelCase__   :							Optional[int]					,       UpperCAmelCase__   :							Tuple					,       UpperCAmelCase__   :							Optional[Any]					,       UpperCAmelCase__   :							Optional[int]     ):
					from accelerate.utils import DummyOptim, DummyScheduler

					SCREAMING_SNAKE_CASE				=				hf_deepspeed_config.config

					# Optimizer + Scheduler
					# Currently supported combos:
					# 1. DS scheduler + DS optimizer: Yes
					# 2. HF scheduler + HF optimizer: Yes
					# 3. DS scheduler + HF optimizer: Yes
					# 4. HF scheduler + DS optimizer: No
					#
					# Unless Offload is enabled in which case it's:
					# 1. DS scheduler + DS optimizer: Yes
					# 2. HF scheduler + HF optimizer: Mostly*
					# 3. DS scheduler + HF optimizer: Mostly*
					# 4. HF scheduler + DS optimizer: No
					#
					# Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

					SCREAMING_SNAKE_CASE				=				None
					if "optimizer" in config:
										if args.adafactor:
															raise ValueError(
															    "--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. "
															    "Only one optimizer can be configured."     )
										SCREAMING_SNAKE_CASE				=				DummyOptim(params=a__     )
					else:
										if hf_deepspeed_config.is_offload():
															logger.info(
															    "Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"
															    " custom optimizer has both CPU and GPU implementation (except LAMB)"     )

										# ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
										# But trainer uses AdamW by default.
										SCREAMING_SNAKE_CASE				=				trainer.create_optimizer()
										# To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
										SCREAMING_SNAKE_CASE				=				True

					SCREAMING_SNAKE_CASE				=				None
					if "scheduler" in config:
										SCREAMING_SNAKE_CASE				=				DummyScheduler(a__     )
					else:
										if isinstance(a__					,       a__     ):
															raise ValueError(
															    "Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. "
															    "Please configure a scheduler in the DeepSpeed config."     )
										SCREAMING_SNAKE_CASE				=				trainer.create_scheduler(num_training_steps=a__					,       optimizer=a__     )

					return optimizer, lr_scheduler





def 		__lowerCamelCase							(UpperCAmelCase__   :							Dict					,       UpperCAmelCase__   :							List[Any]					,       UpperCAmelCase__   :							List[str]=False     ):
					from deepspeed.utils import logger as ds_logger

					SCREAMING_SNAKE_CASE				=				trainer.model
					SCREAMING_SNAKE_CASE				=				trainer.args

					SCREAMING_SNAKE_CASE				=				trainer.accelerator.state.deepspeed_plugin.hf_ds_config

					# resume config update - some bits like `model` and `num_training_steps` only become available during train
					hf_deepspeed_config.trainer_config_finalize(a__					,       a__					,       a__     )

					# set the Deepspeed log level consistent with the Trainer
					ds_logger.setLevel(args.get_process_log_level()     )

					if inference:
										# only Z3 makes sense for the inference
										if not hf_deepspeed_config.is_zeroa():
															raise ValueError("ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config"     )

										# in case the training config is re-used for inference
										hf_deepspeed_config.del_config_sub_tree("optimizer"     )
										hf_deepspeed_config.del_config_sub_tree("lr_scheduler"     )
										SCREAMING_SNAKE_CASE	,      SCREAMING_SNAKE_CASE				=				None, None
										SCREAMING_SNAKE_CASE				=				None
					else:
										SCREAMING_SNAKE_CASE				=				None  # important for when deepspeed_init is used as re-init
										SCREAMING_SNAKE_CASE				=				list(filter(lambda UpperCAmelCase__     : p.requires_grad					,       model.parameters()     )     )
										SCREAMING_SNAKE_CASE	,      SCREAMING_SNAKE_CASE				=				deepspeed_optim_sched(
										    a__					,       a__					,       a__					,       a__					,       a__     )

					# keep for quick debug:
					# from pprint import pprint; pprint(config)

					return optimizer, lr_scheduler





def 		__lowerCamelCase							(UpperCAmelCase__   :							Tuple					,       UpperCAmelCase__   :							Dict     ):
					import glob

					SCREAMING_SNAKE_CASE				=				sorted(glob.glob(F"{checkpoint_path}/global_step*"     )     )

					if len(a__     ) > 0:
										logger.info(F"Attempting to resume from {checkpoint_path}"     )
										# this magically updates self.optimizer and self.lr_scheduler
										SCREAMING_SNAKE_CASE	,      SCREAMING_SNAKE_CASE				=				deepspeed_engine.load_checkpoint(
										    a__					,       load_optimizer_states=a__					,       load_lr_scheduler_states=a__     )
										if load_path is None:
															raise ValueError(F"[deepspeed] failed to resume from checkpoint {checkpoint_path}"     )
					else:
										raise ValueError(F"Can\'t find a valid checkpoint at {checkpoint_path}"     )










'''simple docstring'''
import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import (
    TaBlock,
    TaConfig,
    TaLayerNorm,
)

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin






class        SCREAMING_SNAKE_CASE(				__A   ,      __A   ,      __A     ):

       @register_to_config
       def __init__(    self					,		lowerCamelCase__					,		lowerCamelCase__					,		lowerCamelCase__					,		lowerCamelCase__					,		lowerCamelCase__					,		lowerCamelCase__					,		lowerCamelCase__					,		lowerCamelCase__					,		lowerCamelCase__					,		lowerCamelCase__ = False					,		)    ->					Optional[int]:

           """simple docstring"""



           super().__init__()

           __lowercase							=						nn.Linear(lowerCamelCase__					,		lowerCamelCase__					,		bias=lowerCamelCase__     )

           __lowercase							=						nn.Embedding(lowerCamelCase__					,		lowerCamelCase__     )
           __lowercase							=						False

           __lowercase							=						nn.Dropout(p=lowerCamelCase__     )

           __lowercase							=						TaConfig(
               d_model=lowerCamelCase__					,		num_heads=lowerCamelCase__					,		d_kv=lowerCamelCase__					,		d_ff=lowerCamelCase__					,		feed_forward_proj=lowerCamelCase__					,		dropout_rate=lowerCamelCase__					,		is_decoder=lowerCamelCase__					,		is_encoder_decoder=lowerCamelCase__					,		)
           __lowercase							=						nn.ModuleList()
           for lyr_num in range(lowerCamelCase__     ):
               __lowercase							=						TaBlock(lowerCamelCase__     )
               self.encoders.append(lowerCamelCase__     )

           __lowercase							=						TaLayerNorm(lowerCamelCase__     )
           __lowercase							=						nn.Dropout(p=lowerCamelCase__     )




       def     snake_case__			(    self					,		lowerCamelCase__					,		lowerCamelCase__     )    ->					str:

           """simple docstring"""



           __lowercase							=						self.input_proj(lowerCamelCase__     )

           # terminal relative positional encodings
           __lowercase							=						encoder_inputs.shape[1]
           __lowercase							=						torch.arange(lowerCamelCase__					,		device=encoder_inputs.device     )

           __lowercase							=						encoder_inputs_mask.sum(-1     )
           __lowercase							=						torch.roll(input_positions.unsqueeze(0     )					,		tuple(seq_lens.tolist()     )					,		dims=0     )
           x += self.position_encoding(lowerCamelCase__     )

           __lowercase							=						self.dropout_pre(lowerCamelCase__     )

           # inverted the attention mask
           __lowercase							=						encoder_inputs.size()
           __lowercase							=						self.get_extended_attention_mask(lowerCamelCase__					,		lowerCamelCase__     )

           for lyr in self.encoders:
               __lowercase							=						lyr(lowerCamelCase__					,		lowerCamelCase__     )[0]
           __lowercase							=						self.layer_norm(lowerCamelCase__     )

           return self.dropout_post(lowerCamelCase__     ), encoder_inputs_mask




