
'''simple docstring'''






import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
 from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
 import torch_xla.core.xla_model as xm







def 		A_							(						_lowerCAmelCase :     Dict				):



      """simple docstring"""




      if is_torch_version("<"					,						"2.0.0"				) or not hasattr(_lowerCAmelCase					,						"_dynamo"				):
            return False
      return isinstance(_lowerCAmelCase					,						torch._dynamo.eval_frame.OptimizedModule				)







def 		A_							(						_lowerCAmelCase :     str					,						_lowerCAmelCase :     bool = True				):



      """simple docstring"""




      _lowerCamelCase					: Tuple		    = (torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

      _lowerCamelCase					: Optional[Any]		    = is_compiled_module(_lowerCAmelCase				)
      if is_compiled:
            _lowerCamelCase					: Tuple		    = model
            _lowerCamelCase					: Dict		    = model._orig_mod

      if is_deepspeed_available():
            options += (DeepSpeedEngine,)

      while isinstance(_lowerCAmelCase					,						_lowerCAmelCase				):
            _lowerCamelCase					: Tuple		    = model.module

      if not keep_fpaa_wrapper:
            _lowerCamelCase					: Optional[Any]		    = getattr(_lowerCAmelCase					,						"forward"				)
            _lowerCamelCase					: Any		    = model.__dict__.pop("_original_forward"					,						_lowerCAmelCase				)
            if original_forward is not None:
                  while hasattr(_lowerCAmelCase					,						"__wrapped__"				):
                        _lowerCamelCase					: List[str]		    = forward.__wrapped__
                        if forward == original_forward:
                              break
                  _lowerCamelCase					: str		    = forward
            if getattr(_lowerCAmelCase					,						"_converted_to_transformer_engine"					,						_lowerCAmelCase				):
                  convert_model(_lowerCAmelCase					,						to_transformer_engine=_lowerCAmelCase				)

      if is_compiled:
            _lowerCamelCase					: List[str]		    = model
            _lowerCamelCase					: Optional[int]		    = compiled_model

      return model







def 		A_							(						):



      """simple docstring"""




      PartialState().wait_for_everyone()







def 		A_							(						_lowerCAmelCase :     str					,						_lowerCAmelCase :     Tuple				):



      """simple docstring"""




      if PartialState().distributed_type == DistributedType.TPU:
            xm.save(_lowerCAmelCase					,						_lowerCAmelCase				)
      elif PartialState().local_process_index == 0:
            torch.save(_lowerCAmelCase					,						_lowerCAmelCase				)







@contextmanager
def 		A_							(						**_lowerCAmelCase :     Any				):



      """simple docstring"""




      for key, value in kwargs.items():
            _lowerCamelCase					: str		    = str(_lowerCAmelCase				)

      yield

      for key in kwargs:
            if key.upper() in os.environ:
                  del os.environ[key.upper()]







def 		A_							(						_lowerCAmelCase :     Any				):



      """simple docstring"""




      if not hasattr(_lowerCAmelCase					,						"__qualname__"				) and not hasattr(_lowerCAmelCase					,						"__name__"				):
            _lowerCamelCase					: Any		    = getattr(_lowerCAmelCase					,						"__class__"					,						_lowerCAmelCase				)
      if hasattr(_lowerCAmelCase					,						"__qualname__"				):
            return obj.__qualname__
      if hasattr(_lowerCAmelCase					,						"__name__"				):
            return obj.__name__
      return str(_lowerCAmelCase				)







def 		A_							(						_lowerCAmelCase :     Optional[Any]					,						_lowerCAmelCase :     str				):



      """simple docstring"""




      for key, value in source.items():
            if isinstance(_lowerCAmelCase					,						_lowerCAmelCase				):
                  _lowerCamelCase					: Optional[int]		    = destination.setdefault(_lowerCAmelCase					,						{}				)
                  merge_dicts(_lowerCAmelCase					,						_lowerCAmelCase				)
            else:
                  _lowerCamelCase					: Optional[Any]		    = value

      return destination







def 		A_							(						_lowerCAmelCase :     int = None				):



      """simple docstring"""




      if port is None:
            _lowerCamelCase					: Dict		    = 29500
      with socket.socket(socket.AF_INET					,						socket.SOCK_STREAM				) as s:
            return s.connect_ex(("localhost", port)				) == 0





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCamelCase      :						str							=  logging.get_logger(__name__)

_lowerCamelCase      :						List[str]							=  {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/config.json"""}


class 							UpperCamelCase_       (					UpperCAmelCase__	):

    '''simple docstring'''






    UpperCAmelCase__				    = '''openai-gpt'''
    UpperCAmelCase__				    = {
        '''max_position_embeddings''': '''n_positions''',
        '''hidden_size''': '''n_embd''',
        '''num_attention_heads''': '''n_head''',
        '''num_hidden_layers''': '''n_layer''',
    }


    def __init__(     self :      Union[str, Any]   ,					UpperCAmelCase__ :      Dict=40_478   ,					UpperCAmelCase__ :      str=512   ,					UpperCAmelCase__ :      Union[str, Any]=768   ,					UpperCAmelCase__ :      Optional[Any]=12   ,					UpperCAmelCase__ :      Any=12   ,					UpperCAmelCase__ :      Optional[Any]="gelu"   ,					UpperCAmelCase__ :      Any=0.1   ,					UpperCAmelCase__ :      List[str]=0.1   ,					UpperCAmelCase__ :      Tuple=0.1   ,					UpperCAmelCase__ :      List[str]=1e-5   ,					UpperCAmelCase__ :      int=0.02   ,					UpperCAmelCase__ :      Any="cls_index"   ,					UpperCAmelCase__ :      Optional[Any]=True   ,					UpperCAmelCase__ :      List[Any]=None   ,					UpperCAmelCase__ :      List[Any]=True   ,					UpperCAmelCase__ :      Optional[Any]=0.1   ,					**UpperCAmelCase__ :      Dict   ,					)		->Any:





           '''simple docstring'''

           A__ 					=       vocab_size
           A__ 					=       n_positions
           A__ 					=       n_embd
           A__ 					=       n_layer
           A__ 					=       n_head
           A__ 					=       afn
           A__ 					=       resid_pdrop
           A__ 					=       embd_pdrop
           A__ 					=       attn_pdrop
           A__ 					=       layer_norm_epsilon
           A__ 					=       initializer_range
           A__ 					=       summary_type
           A__ 					=       summary_use_proj
           A__ 					=       summary_activation
           A__ 					=       summary_first_dropout
           A__ 					=       summary_proj_to_labels
           super().__init__(**UpperCAmelCase__)


