





"""simple docstring"""





import baseaa
import io
import json
import os
from copy import deepcopy

from ..optimizer import AcceleratedOptimizer
from ..scheduler import AcceleratedScheduler




class 							snake_case							:

      '''simple docstring'''



      def __init__(							self       :      Optional[int]						,				__lowercase       :      Optional[Any] ):


        '''simple docstring'''




        if isinstance(__lowercase						,				__lowercase ):
          # Don't modify user's data should they want to reuse it (e.g. in tests), because once we
          # modified it, it will not be accepted here again, since `auto` values would have been overridden
          __UpperCAmelCase    :						List[Any]       =						deepcopy(__lowercase )
        elif os.path.exists(__lowercase ):
          with io.open(__lowercase						,				'''r'''						,				encoding='''utf-8''' ) as f:
            __UpperCAmelCase    :						str       =						json.load(__lowercase )
        else:
          try:
            __UpperCAmelCase    :						int       =						baseaa.urlsafe_baadecode(__lowercase ).decode('''utf-8''' )
            __UpperCAmelCase    :						Optional[int]       =						json.loads(__lowercase )
          except (UnicodeDecodeError, AttributeError, ValueError):
            raise ValueError(
                f'''Expected a string path to an existing deepspeed config, or a dictionary, or a base64 encoded string. Received: {config_file_or_dict}''' )

        __UpperCAmelCase    :						int       =						config

        self.set_stage_and_offload()

      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						str       =						self.get_value('''zero_optimization.stage'''						,				-1 )

        # offload
        __UpperCAmelCase    :						Any       =						False
        if self.is_zeroa() or self.is_zeroa():
          __UpperCAmelCase    :						Optional[Any]       =						set(['''cpu''', '''nvme'''] )
          __UpperCAmelCase    :						Optional[Any]       =						set(
              [
                  self.get_value('''zero_optimization.offload_optimizer.device''' ),
                  self.get_value('''zero_optimization.offload_param.device''' ),
              ] )
          if len(offload_devices & offload_devices_valid ) > 0:
            __UpperCAmelCase    :						Tuple       =						True

      def       A_					(							self       :      List[Any]						,				__lowercase       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						self.config

        # find the config node of interest if it exists
        __UpperCAmelCase    :						int       =						ds_key_long.split('''.''' )
        __UpperCAmelCase    :						Dict       =						nodes.pop()
        for node in nodes:
          __UpperCAmelCase    :						Optional[Any]       =						config.get(__lowercase )
          if config is None:
            return None, ds_key

        return config, ds_key

      def       A_					(							self       :      Optional[Any]						,				__lowercase       :      Dict						,				__lowercase       :      List[str]=None ):


        '''simple docstring'''




        __UpperCAmelCase    :						Union[str, Any]       =						self.find_config_node(__lowercase )
        if config is None:
          return default
        return config.get(__lowercase						,				__lowercase )

      def       A_					(							self       :      Optional[Any]						,				__lowercase       :      Optional[Any]						,				__lowercase       :      Optional[int]=False ):


        '''simple docstring'''




        __UpperCAmelCase    :						Union[str, Any]       =						self.config

        # find the config node of interest if it exists
        __UpperCAmelCase    :						List[str]       =						ds_key_long.split('''.''' )
        for node in nodes:
          __UpperCAmelCase    :						Dict       =						config
          __UpperCAmelCase    :						Union[str, Any]       =						config.get(__lowercase )
          if config is None:
            if must_exist:
              raise ValueError(f'''Can\'t find {ds_key_long} entry in the config: {self.config}''' )
            else:
              return

        # if found remove it
        if parent_config is not None:
          parent_config.pop(__lowercase )

      def       A_					(							self       :      Optional[int]						,				__lowercase       :      Optional[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						str       =						self.get_value(__lowercase )
        return False if value is None else bool(__lowercase )

      def       A_					(							self       :      List[Any]						,				__lowercase       :      Optional[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						self.get_value(__lowercase )
        return False if value is None else not bool(__lowercase )

      def       A_					(							self       :      Tuple ):


        '''simple docstring'''




        return self._stage == 2

      def       A_					(							self       :      Union[str, Any] ):


        '''simple docstring'''




        return self._stage == 3







      def       A_					(							self       :      Any ):


        '''simple docstring'''




        return self._offload




class 							snake_case							:

      '''simple docstring'''



      def __init__(							self       :      List[Any]						,				__lowercase       :      Optional[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						engine







      def       A_					(							self       :      List[Any]						,				__lowercase       :      Optional[int]						,				**__lowercase       :      List[Any] ):


        '''simple docstring'''




        self.engine.backward(__lowercase						,				**__lowercase )

        # Deepspeed's `engine.step` performs the following operations:
        # - gradient accumulation check
        # - gradient clipping
        # - optimizer step
        # - zero grad
        # - checking overflow
        # - lr_scheduler step (only if engine.lr_scheduler is not None)
        self.engine.step()
        # and this plugin overrides the above calls with no-ops when Accelerate runs under
        # Deepspeed, but allows normal functionality for non-Deepspeed cases thus enabling a simple
        # training loop that works transparently under many training regimes.




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      def __init__(							self       :      Union[str, Any]						,				__lowercase       :      str ):


        '''simple docstring'''




        super().__init__(__lowercase						,				device_placement=__lowercase						,				scaler=__lowercase )
        __UpperCAmelCase    :						Dict       =						hasattr(self.optimizer						,				'''overflow''' )

      def       A_					(							self       :      List[Any]						,				__lowercase       :      str=None ):


        '''simple docstring'''




        pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed

      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed







      @property
      def       A_					(							self       :      Any ):


        '''simple docstring'''




        if self.__has_overflow__:
          return self.optimizer.overflow
        return False




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      def __init__(							self       :      int						,				__lowercase       :      List[str]						,				__lowercase       :      Any ):


        '''simple docstring'''




        super().__init__(__lowercase						,				__lowercase )







      def       A_					(							self       :      str ):


        '''simple docstring'''




        pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed




class 							snake_case							:

      '''simple docstring'''



      def __init__(							self       :      Tuple						,				__lowercase       :      Union[str, Any]						,				__lowercase       :      Union[str, Any]=0.0_0_1						,				__lowercase       :      Dict=0						,				**__lowercase       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[int]       =						params
        __UpperCAmelCase    :						Optional[int]       =						lr
        __UpperCAmelCase    :						List[Any]       =						weight_decay
        __UpperCAmelCase    :						Any       =						kwargs




class 							snake_case							:

      '''simple docstring'''



      def __init__(							self       :      List[str]						,				__lowercase       :      List[Any]						,				__lowercase       :      Optional[Any]=None						,				__lowercase       :      List[Any]=0						,				**__lowercase       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						optimizer
        __UpperCAmelCase    :						List[Any]       =						total_num_steps
        __UpperCAmelCase    :						List[str]       =						warmup_num_steps
        __UpperCAmelCase    :						Tuple       =						kwargs










"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowercase__				:Tuple				   =       {
    'configuration_xlm': ['XLM_PRETRAINED_CONFIG_ARCHIVE_MAP', 'XLMConfig', 'XLMOnnxConfig'],
    'tokenization_xlm': ['XLMTokenizer'],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase__				:Optional[Any]				   =       [
      'XLM_PRETRAINED_MODEL_ARCHIVE_LIST',
      'XLMForMultipleChoice',
      'XLMForQuestionAnswering',
      'XLMForQuestionAnsweringSimple',
      'XLMForSequenceClassification',
      'XLMForTokenClassification',
      'XLMModel',
      'XLMPreTrainedModel',
      'XLMWithLMHeadModel',
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase__				:List[Any]				   =       [
      'TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST',
      'TFXLMForMultipleChoice',
      'TFXLMForQuestionAnsweringSimple',
      'TFXLMForSequenceClassification',
      'TFXLMForTokenClassification',
      'TFXLMMainLayer',
      'TFXLMModel',
      'TFXLMPreTrainedModel',
      'TFXLMWithLMHeadModel',
  ]


if TYPE_CHECKING:
  from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, XLMOnnxConfig
  from .tokenization_xlm import XLMTokenizer

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_xlm import (
        XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        XLMForMultipleChoice,
        XLMForQuestionAnswering,
        XLMForQuestionAnsweringSimple,
        XLMForSequenceClassification,
        XLMForTokenClassification,
        XLMModel,
        XLMPreTrainedModel,
        XLMWithLMHeadModel,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_xlm import (
        TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMForMultipleChoice,
        TFXLMForQuestionAnsweringSimple,
        TFXLMForSequenceClassification,
        TFXLMForTokenClassification,
        TFXLMMainLayer,
        TFXLMModel,
        TFXLMPreTrainedModel,
        TFXLMWithLMHeadModel,
    )

else:
  import sys

  lowercase__				:List[Any]				   =       _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)