






import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
							import torch

lowercase     :							Dict	     =			logging.get_logger(__name__)

def    lowerCAmelCase__		(			):
		return importlib.util.find_spec("deepspeed"			) is not None


if is_accelerate_available() and is_deepspeed_available():
							from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
							# Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
							# Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
							from builtins import object as DeepSpeedConfig




class 						UpperCAmelCase_  (		lowerCamelCase_    ):


						'''simple docstring'''




						def __init__(		self						, _SCREAMING_SNAKE_CASE )       -> Optional[Any]:
								set_hf_deepspeed_config(self )
								dep_version_check("accelerate" )
								dep_version_check("deepspeed" )
								super().__init__(snake_case_ )




class 						UpperCAmelCase_  (		lowerCamelCase_    ):


						'''simple docstring'''




						def __init__(		self						, _SCREAMING_SNAKE_CASE )       -> Tuple:
								super().__init__(snake_case_ )
								snake_case_ :							Tuple         =				None
								snake_case_ :							Any         =				[]






						def 					_lowerCAmelCase				(		self )       -> int:

								if self._dtype is None:
										raise ValueError("trainer_config_process() wasn\'t called yet to tell dtype" )
								return self._dtype






						def 					_lowerCAmelCase				(		self						, _SCREAMING_SNAKE_CASE )       -> int:
								snake_case_ :							Tuple         =				self.get_value(snake_case_ )
								if val is None:
										return False
								else:
										return val == "auto"






						def 					_lowerCAmelCase				(		self						, _SCREAMING_SNAKE_CASE						, _SCREAMING_SNAKE_CASE						, _SCREAMING_SNAKE_CASE=None						, _SCREAMING_SNAKE_CASE=True )       -> List[Any]:
								snake_case_ :							int         =				self.find_config_node(snake_case_ )
								if config is None:
										return

								if config.get(snake_case_ ) == "auto":
										snake_case_ :							int         =				hf_val
										return

								if not must_match:
										return

								snake_case_ :							Optional[Any]         =				config.get(snake_case_ )
								if ds_val is not None and ds_val != hf_val:
										self.mismatches.append(f'''- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}''' )

						A   :      Tuple						 =    partialmethod(lowerCamelCase_    ,     must_match=lowerCamelCase_    )






						def 					_lowerCAmelCase				(		self						, _SCREAMING_SNAKE_CASE )       -> Union[str, Any]:
								snake_case_ :							Union[str, Any]         =				args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
								self.fill_match(
								    "train_micro_batch_size_per_gpu"						, args.per_device_train_batch_size						, "per_device_train_batch_size" )
								self.fill_match("gradient_accumulation_steps"						, args.gradient_accumulation_steps						, "gradient_accumulation_steps" )
								self.fill_match("train_batch_size"						, snake_case_						, "train_batch_size (calculated)" )
								self.fill_match("gradient_clipping"						, args.max_grad_norm						, "max_grad_norm" )

								self.fill_match("optimizer.params.lr"						, args.learning_rate						, "learning_rate" )
								self.fill_match("optimizer.params.betas"						, [args.adam_betaa, args.adam_betaa]						, "adam_beta1+adam_beta2" )
								self.fill_match("optimizer.params.eps"						, args.adam_epsilon						, "adam_epsilon" )
								self.fill_match("optimizer.params.weight_decay"						, args.weight_decay						, "weight_decay" )

								self.fill_only("scheduler.params.warmup_min_lr"						, 0 )  # not a trainer arg
								self.fill_match("scheduler.params.warmup_max_lr"						, args.learning_rate						, "learning_rate" )
								# total_num_steps - will get set in trainer_config_finalize

								# fp16
								if args.fpaa or args.fpaa_full_eval:
										snake_case_ :							Any         =				'apex' if args.fpaa_backend == 'apex' else 'amp'
								else:
										snake_case_ :							Union[str, Any]         =				None

								if args.save_on_each_node:
										# deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
										snake_case_ :							List[str]         =				self.config.get("checkpoint"						, {} )
										snake_case_ :							str         =				args.save_on_each_node

								# amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
								# any here unless the user did the work
								self.fill_match(
								    "fp16.enabled"						, ((args.fpaa or args.fpaa_full_eval) and fpaa_backend == "amp")						, "fp16|fp16_full_eval+fp16_backend(amp)"						, )

								# apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
								# ZeRO features
								self.fill_match("amp.enabled"						, fpaa_backend == "apex"						, "fp16+fp16_backend(apex)" )
								self.fill_match("amp.opt_level"						, args.fpaa_opt_level						, "fp16_opt_level" )

								self.fill_match("bf16.enabled"						, (args.bfaa or args.bfaa_full_eval)						, "bf16|bf16_full_eval" )

								# deepspeed's default mode is fp16 unless there is a config that says differently
								if self.is_true("bf16.enabled" ):
										snake_case_ :							Dict         =				torch.bfloataa
								elif self.is_false("fp16.enabled" ):
										snake_case_ :							List[Any]         =				torch.floataa
								else:
										snake_case_ :							Optional[int]         =				torch.floataa






						def 					_lowerCAmelCase				(		self						, _SCREAMING_SNAKE_CASE						, _SCREAMING_SNAKE_CASE						, _SCREAMING_SNAKE_CASE )       -> List[str]:
								snake_case_ :							List[str]         =				[
								    'zero_optimization.reduce_bucket_size',
								    'zero_optimization.stage3_prefetch_bucket_size',
								    'zero_optimization.stage3_param_persistence_threshold',
								]
								snake_case_ :							str         =				[x for x in hidden_size_based_keys if self.is_auto(snake_case_ )]

								if len(snake_case_ ) > 0:
										if hasattr(model.config						, "hidden_size" ):
												snake_case_ :							List[Any]         =				model.config.hidden_size
										elif hasattr(model.config						, "hidden_sizes" ):
												# if there are many hidden sizes pick the largest one
												snake_case_ :							List[str]         =				max(model.config.hidden_sizes )
										else:
												raise ValueError(
												    "The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, "
												    "therefore it\'s not possible to automatically fill out the following `auto` entries "
												    f'''in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing '''
												    "`auto` values for these keys with an integer value of your choice." )

										self.fill_only("zero_optimization.reduce_bucket_size"						, hidden_size * hidden_size )
										if self.is_zeroa():
												# automatically assign the optimal config values based on model config
												self.fill_only("zero_optimization.stage3_prefetch_bucket_size"						, 0.9 * hidden_size * hidden_size )
												self.fill_only("zero_optimization.stage3_param_persistence_threshold"						, 10 * hidden_size )

        # scheduler
								self.fill_match("scheduler.params.total_num_steps"						, snake_case_						, "num_training_steps (calculated)" )
								self.fill_match("scheduler.params.warmup_num_steps"						, args.get_warmup_steps(snake_case_ )						, "warmup_steps" )

								if len(self.mismatches ) > 0:
										snake_case_ :							List[Any]         =				'\n'.join(self.mismatches )
										raise ValueError(
										    "Please correct the following DeepSpeed config values that mismatch TrainingArguments"
										    f''' values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'.''' )




lowercase     :							List[str]	     =			None

def    lowerCAmelCase__		(			_a				:	Optional[int]			):
		global _hf_deepspeed_config_weak_ref
		# will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
		snake_case_ :							List[str]         =				weakref.ref(lowerCamelCase_			)

def    lowerCAmelCase__		(			):
		global _hf_deepspeed_config_weak_ref
		snake_case_ :							int         =				None

def    lowerCAmelCase__		(			):





		if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
				return _hf_deepspeed_config_weak_ref().is_zeroa()
		else:
				return False

def    lowerCAmelCase__		(			):





		if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
				return _hf_deepspeed_config_weak_ref().config
		else:
				return None

def    lowerCAmelCase__		(			_a				:	Optional[int]     ,					_a				:	Optional[Any]     ,					_a				:	int     ,					_a				:	Union[str, Any]     ,					_a				:	Union[str, Any]			):
		from accelerate.utils import DummyOptim, DummyScheduler

		snake_case_ :							Optional[Any]         =				hf_deepspeed_config.config

		# Optimizer + Scheduler
		# Currently supported combos:
		# 1. DS scheduler + DS optimizer: Yes
		# 2. HF scheduler + HF optimizer: Yes
		# 3. DS scheduler + HF optimizer: Yes
		# 4. HF scheduler + DS optimizer: No
		#
		# Unless Offload is enabled in which case it's:
		# 1. DS scheduler + DS optimizer: Yes
		# 2. HF scheduler + HF optimizer: Mostly*
		# 3. DS scheduler + HF optimizer: Mostly*
		# 4. HF scheduler + DS optimizer: No
		#
		# Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

		snake_case_ :							int         =				None
		if "optimizer" in config:
				if args.adafactor:
						raise ValueError(
						    "--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. "
						    "Only one optimizer can be configured."			)
				snake_case_ :							int         =				DummyOptim(params=lowerCamelCase_			)
		else:
				if hf_deepspeed_config.is_offload():
						logger.info(
						    "Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"
						    " custom optimizer has both CPU and GPU implementation (except LAMB)"			)

				# ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
				# But trainer uses AdamW by default.
				snake_case_ :							Tuple         =				trainer.create_optimizer()
				# To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
				snake_case_ :							Any         =				True

		snake_case_ :							int         =				None
		if "scheduler" in config:
				snake_case_ :							Union[str, Any]         =				DummyScheduler(lowerCamelCase_			)
		else:
				if isinstance(lowerCamelCase_     ,					lowerCamelCase_			):
						raise ValueError(
						    "Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. "
						    "Please configure a scheduler in the DeepSpeed config."			)
				snake_case_ :							List[Any]         =				trainer.create_scheduler(num_training_steps=lowerCamelCase_     ,					optimizer=lowerCamelCase_			)

		return optimizer, lr_scheduler

def    lowerCAmelCase__		(			_a				:	List[str]     ,					_a				:	str     ,					_a				:	str=False			):
		from deepspeed.utils import logger as ds_logger

		snake_case_ :							List[str]         =				trainer.model
		snake_case_ :							List[Any]         =				trainer.args

		snake_case_ :							Optional[int]         =				trainer.accelerator.state.deepspeed_plugin.hf_ds_config

		# resume config update - some bits like `model` and `num_training_steps` only become available during train
		hf_deepspeed_config.trainer_config_finalize(lowerCamelCase_     ,					lowerCamelCase_     ,					lowerCamelCase_			)

		# set the Deepspeed log level consistent with the Trainer
		ds_logger.setLevel(args.get_process_log_level()			)

		if inference:
				# only Z3 makes sense for the inference
				if not hf_deepspeed_config.is_zeroa():
						raise ValueError("ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config"			)

				# in case the training config is re-used for inference
				hf_deepspeed_config.del_config_sub_tree("optimizer"			)
				hf_deepspeed_config.del_config_sub_tree("lr_scheduler"			)
				snake_case_ :							List[Any]         =				None, None
				snake_case_ :							str         =				None
		else:
				snake_case_ :							Union[str, Any]         =				None  # important for when deepspeed_init is used as re-init
				snake_case_ :							Tuple         =				list(filter(lambda _a			: p.requires_grad     ,					model.parameters()			)			)
				snake_case_ :							List[str]         =				deepspeed_optim_sched(
				    lowerCamelCase_     ,					lowerCamelCase_     ,					lowerCamelCase_     ,					lowerCamelCase_     ,					lowerCamelCase_			)

		# keep for quick debug:
		# from pprint import pprint; pprint(config)

		return optimizer, lr_scheduler

def    lowerCAmelCase__		(			_a				:	Tuple     ,					_a				:	int			):
		import glob

		snake_case_ :							Dict         =				sorted(glob.glob(F'''{checkpoint_path}/global_step*'''			)			)

		if len(lowerCamelCase_			) > 0:
				logger.info(F'''Attempting to resume from {checkpoint_path}'''			)
				# this magically updates self.optimizer and self.lr_scheduler
				snake_case_ :							str         =				deepspeed_engine.load_checkpoint(
				    lowerCamelCase_     ,					load_optimizer_states=lowerCamelCase_     ,					load_lr_scheduler_states=lowerCamelCase_			)
				if load_path is None:
						raise ValueError(F'''[deepspeed] failed to resume from checkpoint {checkpoint_path}'''			)
		else:
				raise ValueError(F'''Can\'t find a valid checkpoint at {checkpoint_path}'''			)

'''simple docstring'''


from __future__ import annotations

from collections import deque
from enum import Enum
from math import atana, degrees
from sys import maxsize






class      __SCREAMING_SNAKE_CASE						(       lowerCamelCase_       ):







  '''simple docstring'''






  lowerCamelCase_		:Union[str, Any]						    =		1
  lowerCamelCase_		:List[Any]						    =		2
  lowerCamelCase_		:Optional[Any]						    =		3


  def __repr__( self						):




       '''simple docstring'''

       return F'''{self.__class__.__name__}.{self.name}'''







def 					_lowerCamelCase    (      lowerCamelCase_					:							tuple[int, int]	,     lowerCamelCase_					:							int	,     lowerCamelCase_					:							int   ):



     """simple docstring"""





     UpperCAmelCase_   ,			UpperCAmelCase_							:   Any   =      point
     return degrees(atana(y - miny	,     x - minx   )   )


def 					_lowerCamelCase    (      lowerCamelCase_					:							tuple[int, int]	,     lowerCamelCase_					:							tuple[int, int]	,     lowerCamelCase_					:							tuple[int, int]   ):



     """simple docstring"""





     UpperCAmelCase_   ,			UpperCAmelCase_							:   str   =      starting
     UpperCAmelCase_   ,			UpperCAmelCase_							:   Optional[int]   =      via
     UpperCAmelCase_   ,			UpperCAmelCase_							:   Dict   =      target
     UpperCAmelCase_							:   Optional[Any]   =      degrees(atana(ya - ya	,     xa - xa   )   )
     via_angle %= 360
     UpperCAmelCase_							:   Optional[Any]   =      degrees(atana(ya - ya	,     xa - xa   )   )
     target_angle %= 360
     # t-
     #  \ \
     #   \ v
     #    \|
     #     s
     # via_angle is always lower than target_angle, if direction is left.
     # If they are same, it means they are on a same line of convex hull.
     if target_angle > via_angle:
          return Direction.left
     elif target_angle == via_angle:
          return Direction.straight
     else:
          return Direction.right





def 					_lowerCamelCase    (      lowerCamelCase_					:							list[tuple[int, int]]   ):



     """simple docstring"""





     if len(lowerCamelCase_   ) <= 2:
          # There is no convex hull
          raise ValueError('graham_scan: argument must contain more than 3 points.'   )
     if len(lowerCamelCase_   ) == 3:
          return points
     # find the lowest and the most left point
     UpperCAmelCase_							:   Dict   =      0
     UpperCAmelCase_   ,			UpperCAmelCase_							:   Union[str, Any]   =      maxsize, maxsize
     for i, point in enumerate(lowerCamelCase_   ):
          UpperCAmelCase_							:   Union[str, Any]   =      point[0]
          UpperCAmelCase_							:   List[Any]   =      point[1]
          if y < miny:
               UpperCAmelCase_							:   Dict   =      y
               UpperCAmelCase_							:   Optional[int]   =      x
               UpperCAmelCase_							:   Optional[Any]   =      i
          if y == miny and x < minx:
               UpperCAmelCase_							:   Union[str, Any]   =      x
               UpperCAmelCase_							:   Dict   =      i

    # remove the lowest and the most left point from points for preparing for sort
     points.pop(lowerCamelCase_   )

     UpperCAmelCase_							:   List[str]   =      sorted(lowerCamelCase_	,     key=lambda lowerCamelCase_   : angle_comparer(lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_   )   )
     # This insert actually costs complexity,
     # and you should instead add (minx, miny) into stack later.
     # I'm using insert just for easy understanding.
     sorted_points.insert(0	,     (minx, miny)   )

     UpperCAmelCase_							:   deque[tuple[int, int]]   =      deque()
     stack.append(sorted_points[0]   )
     stack.append(sorted_points[1]   )
     stack.append(sorted_points[2]   )
     # In any ways, the first 3 points line are towards left.
     # Because we sort them the angle from minx, miny.
     UpperCAmelCase_							:   List[Any]   =      Direction.left

     for i in range(3	,     len(lowerCamelCase_   )   ):
          while True:
               UpperCAmelCase_							:   Optional[Any]   =      stack[-2]
               UpperCAmelCase_							:   Optional[int]   =      stack[-1]
               UpperCAmelCase_							:   str   =      sorted_points[i]
               UpperCAmelCase_							:   Union[str, Any]   =      check_direction(lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_   )

               if next_direction == Direction.left:
                    UpperCAmelCase_							:   Union[str, Any]   =      Direction.left
                    break
               if next_direction == Direction.straight:
                    if current_direction == Direction.left:
                         # We keep current_direction as left.
                         # Because if the straight line keeps as straight,
                         # we want to know if this straight line is towards left.
                         break
                    elif current_direction == Direction.right:
                         # If the straight line is towards right,
                         # every previous points on those straigh line is not convex hull.
                         stack.pop()
               if next_direction == Direction.right:
                    stack.pop()
          stack.append(sorted_points[i]   )
     return list(lowerCamelCase_   )






