





"""simple docstring"""


import importlib
import importlib.metadata
import os
import warnings
from distutils.util import strtobool
from functools import lru_cache

import torch
from packaging import version
from packaging.version import parse

from .environment import parse_flag_from_env
from .versions import compare_versions, is_torch_version


try:
      import torch_xla.core.xla_model as xm  # noqa: F401

      A_     :      Union[str, Any]								=True
except ImportError:
      A_     :      str								=False


# Cache this result has it's a C FFI call which can be pretty time-consuming
A_     :      Optional[int]								=torch.distributed.is_available()





def SCREAMING_SNAKE_CASE_   (       snake_case   :					str		)->		Dict:
       # Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version
       _lowerCamelCase    	=					importlib.util.find_spec(_UpperCAmelCase		) is not None
       if package_exists:
              try:
                     _lowerCamelCase    	=					importlib.metadata.metadata(_UpperCAmelCase		)
                     return True
              except importlib.metadata.PackageNotFoundError:
                     return False





def SCREAMING_SNAKE_CASE_   (       )->		bool:
       return _torch_distributed_available





def SCREAMING_SNAKE_CASE_   (       )->		str:
       try:
              pass
       except ImportError:
              print(
                  'Intel(R) oneCCL Bindings for PyTorch* is required to run DDP on Intel(R) GPUs, but it is not'
                  ' detected. If you see \"ValueError: Invalid backend: \'ccl\'\" error, please install Intel(R) oneCCL'
                  ' Bindings for PyTorch*.'		)
       return (
           importlib.util.find_spec('torch_ccl'		) is not None
           or importlib.util.find_spec('oneccl_bindings_for_pytorch'		) is not None
       )





def SCREAMING_SNAKE_CASE_   (       )->		Union[str, Any]:
       return importlib.metadata.version('oneccl_bind_pt'		)





def SCREAMING_SNAKE_CASE_   (       )->		List[str]:
       return _is_package_available('transformer_engine'		)





@lru_cache
def SCREAMING_SNAKE_CASE_   (       snake_case   :					Any=True		)->		str:
       # Due to bugs on the amp series GPUs, we disable torch-xla on them
       if torch.cuda.is_available():
              return False
       if _tpu_available and check_device:
              try:
                     # Will raise a RuntimeError if no XLA configuration is found
                     _lowerCamelCase    	=					xm.xla_device()
                     return True
              except RuntimeError:
                     return False
       return _tpu_available





def SCREAMING_SNAKE_CASE_   (       )->		Tuple:
       return _is_package_available('deepspeed'		)





def SCREAMING_SNAKE_CASE_   (       snake_case   :					List[str]=False		)->		str:
       if is_tpu_available():
              return not ignore_tpu
       if torch.cuda.is_available():
              return torch.cuda.is_bfaa_supported()
       if is_npu_available():
              return False
       return True





def SCREAMING_SNAKE_CASE_   (       )->		Any:
       _lowerCamelCase    	=					_is_package_available('bitsandbytes'		)
       if package_exists:
              _lowerCamelCase    	=					version.parse(importlib.metadata.version('bitsandbytes'		)		)
              return compare_versions(_UpperCAmelCase						,  '>='						,  '0.39.0'		)
       return False





def SCREAMING_SNAKE_CASE_   (       )->		List[Any]:
       _lowerCamelCase    	=					_is_package_available('bitsandbytes'		)
       if package_exists:
              _lowerCamelCase    	=					version.parse(importlib.metadata.version('bitsandbytes'		)		)
              return compare_versions(_UpperCAmelCase						,  '>='						,  '0.37.2'		)
       return False





def SCREAMING_SNAKE_CASE_   (       )->		int:
       return _is_package_available('bitsandbytes'		)





def SCREAMING_SNAKE_CASE_   (       )->		Dict:
       if strtobool(os.environ.get('ACCELERATE_USE_MEGATRON_LM'						,  'False'		)		) == 1:
              _lowerCamelCase    	=					importlib.util.find_spec('megatron'		) is not None
              if package_exists:
                     try:
                            _lowerCamelCase    	=					parse(importlib.metadata.version('megatron-lm'		)		)
                            return compare_versions(_UpperCAmelCase						,  '>='						,  '2.2.0'		)
                     except Exception as e:
                            warnings.warn(f'Parse Megatron version failed. Exception:{e}'		)
                            return False





def SCREAMING_SNAKE_CASE_   (       )->		Dict:
       return _is_package_available('safetensors'		)





def SCREAMING_SNAKE_CASE_   (       )->		int:
       return _is_package_available('transformers'		)





def SCREAMING_SNAKE_CASE_   (       )->		Any:
       return _is_package_available('datasets'		)





def SCREAMING_SNAKE_CASE_   (       )->		Optional[int]:
       return _is_package_available('aim'		)





def SCREAMING_SNAKE_CASE_   (       )->		Union[str, Any]:
       return _is_package_available('tensorboard'		) or _is_package_available('tensorboardX'		)





def SCREAMING_SNAKE_CASE_   (       )->		Tuple:
       return _is_package_available('wandb'		)





def SCREAMING_SNAKE_CASE_   (       )->		Tuple:
       return _is_package_available('comet_ml'		)





def SCREAMING_SNAKE_CASE_   (       )->		Union[str, Any]:
       return _is_package_available('boto3'		)





def SCREAMING_SNAKE_CASE_   (       )->		Any:
       if _is_package_available('rich'		):
              if "ACCELERATE_DISABLE_RICH" in os.environ:
                     warnings.warn(
                         '`ACCELERATE_DISABLE_RICH` is deprecated and will be removed in v0.22.0 and deactivated by default. Please use `ACCELERATE_ENABLE_RICH` if you wish to use `rich`.'		)
                     return not parse_flag_from_env('ACCELERATE_DISABLE_RICH'						,  _UpperCAmelCase		)
              return parse_flag_from_env('ACCELERATE_ENABLE_RICH'						,  _UpperCAmelCase		)
       return False





def SCREAMING_SNAKE_CASE_   (       )->		Tuple:
       return _is_package_available('sagemaker'		)





def SCREAMING_SNAKE_CASE_   (       )->		Union[str, Any]:
       return _is_package_available('tqdm'		)





def SCREAMING_SNAKE_CASE_   (       )->		str:
       return _is_package_available('mlflow'		)





def SCREAMING_SNAKE_CASE_   (       )->		Optional[int]:
       return is_torch_version('>='						,  '1.12'		) and torch.backends.mps.is_available() and torch.backends.mps.is_built()





def SCREAMING_SNAKE_CASE_   (       )->		Any:
       def get_major_and_minor_from_version(snake_case   :					Tuple		):
              return str(version.parse(_UpperCAmelCase		).major		) + "." + str(version.parse(_UpperCAmelCase		).minor		)

       _lowerCamelCase    	=					importlib.metadata.version('torch'		)
       if importlib.util.find_spec('intel_extension_for_pytorch'		) is None:
              return False
       _lowerCamelCase    	=					'N/A'
       try:
              _lowerCamelCase    	=					importlib.metadata.version('intel_extension_for_pytorch'		)
       except importlib.metadata.PackageNotFoundError:
              return False
       _lowerCamelCase    	=					get_major_and_minor_from_version(_torch_version		)
       _lowerCamelCase    	=					get_major_and_minor_from_version(_ipex_version		)
       if torch_major_and_minor != ipex_major_and_minor:
              warnings.warn(
                  f'Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,'
                  f' but PyTorch {_torch_version} is found. Please switch to the matching version and run again.'		)
              return False
       return True





@lru_cache
def SCREAMING_SNAKE_CASE_   (       snake_case   :					Optional[int]=False		)->		Union[str, Any]:
       if importlib.util.find_spec('torch'		) is None or importlib.util.find_spec('torch_npu'		) is None:
              return False

       import torch
       import torch_npu  # noqa: F401

       if check_device:
              try:
                     # Will raise a RuntimeError if no NPU is found
                     _lowerCamelCase    	=					torch.npu.device_count()
                     return torch.npu.is_available()
              except RuntimeError:
                     return False
       return hasattr(_UpperCAmelCase						,  'npu'		) and torch.npu.is_available()





@lru_cache
def SCREAMING_SNAKE_CASE_   (       snake_case   :					str=False		)->		str:
       if not parse_flag_from_env('ACCELERATE_USE_XPU'						,  default=_UpperCAmelCase		):
              return False
       if is_ipex_available():
              import torch

              if is_torch_version('<='						,  '1.12'		):
                     return False
       else:
              return False

       import intel_extension_for_pytorch  # noqa: F401

       if check_device:
              try:
                     # Will raise a RuntimeError if no XPU  is found
                     _lowerCamelCase    	=					torch.xpu.device_count()
                     return torch.xpu.is_available()
              except RuntimeError:
                     return False
       return hasattr(_UpperCAmelCase						,  'xpu'		) and torch.xpu.is_available()













from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


UpperCAmelCase__  :	str     =logging.get_logger(__name__)

UpperCAmelCase__  :	Optional[int]     ={
    '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/config.json''',
    '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/config.json''',
    '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/config.json''',
    '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/config.json''',
    '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/config.json''',
}

class 					__A   (  a			):
				__A            = """gpt2"""
				__A            = ["""past_key_values"""]
				__A            = {
				    """hidden_size""": """n_embd""",
				    """max_position_embeddings""": """n_positions""",
				    """num_attention_heads""": """n_head""",
				    """num_hidden_layers""": """n_layer""",
				}


				def __init__(						self      ,       UpperCAmelCase_=50257      ,       UpperCAmelCase_=1024      ,       UpperCAmelCase_=768      ,       UpperCAmelCase_=12      ,       UpperCAmelCase_=12      ,       UpperCAmelCase_=None      ,       UpperCAmelCase_="gelu_new"      ,       UpperCAmelCase_=0.1      ,       UpperCAmelCase_=0.1      ,       UpperCAmelCase_=0.1      ,       UpperCAmelCase_=1E-5      ,       UpperCAmelCase_=0.0_2      ,       UpperCAmelCase_="cls_index"      ,       UpperCAmelCase_=True      ,       UpperCAmelCase_=None      ,       UpperCAmelCase_=True      ,       UpperCAmelCase_=0.1      ,       UpperCAmelCase_=True      ,       UpperCAmelCase_=True      ,       UpperCAmelCase_=50256      ,       UpperCAmelCase_=50256      ,       UpperCAmelCase_=False      ,       UpperCAmelCase_=False      ,       **UpperCAmelCase_      ,       ):
								lowerCamelCase		       =vocab_size
								lowerCamelCase		       =n_positions
								lowerCamelCase		       =n_embd
								lowerCamelCase		       =n_layer
								lowerCamelCase		       =n_head
								lowerCamelCase		       =n_inner
								lowerCamelCase		       =activation_function
								lowerCamelCase		       =resid_pdrop
								lowerCamelCase		       =embd_pdrop
								lowerCamelCase		       =attn_pdrop
								lowerCamelCase		       =layer_norm_epsilon
								lowerCamelCase		       =initializer_range
								lowerCamelCase		       =summary_type
								lowerCamelCase		       =summary_use_proj
								lowerCamelCase		       =summary_activation
								lowerCamelCase		       =summary_first_dropout
								lowerCamelCase		       =summary_proj_to_labels
								lowerCamelCase		       =scale_attn_weights
								lowerCamelCase		       =use_cache
								lowerCamelCase		       =scale_attn_by_inverse_layer_idx
								lowerCamelCase		       =reorder_and_upcast_attn

								lowerCamelCase		       =bos_token_id
								lowerCamelCase		       =eos_token_id

								super().__init__(bos_token_id=UpperCAmelCase_      ,       eos_token_id=UpperCAmelCase_      ,       **UpperCAmelCase_							)




class 					__A   (  a			):


				def __init__(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = "default"      ,       UpperCAmelCase_ = None      ,       UpperCAmelCase_ = False      ,       ):
								super().__init__(UpperCAmelCase_      ,       task=UpperCAmelCase_      ,       patching_specs=UpperCAmelCase_      ,       use_past=UpperCAmelCase_							)
								if not getattr(self._config      ,       """pad_token_id"""      ,       UpperCAmelCase_							):
												# TODO: how to do that better?
												lowerCamelCase		       =0


				@property
				def _snake_case				(						self							):
								lowerCamelCase		       =OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}							)
								if self.use_past:
												self.fill_with_past_key_values_(UpperCAmelCase_      ,       direction="""inputs"""							)
												lowerCamelCase		       ={0: """batch""", 1: """past_sequence + sequence"""}
								else:
												lowerCamelCase		       ={0: """batch""", 1: """sequence"""}

								return common_inputs


				@property
				def _snake_case				(						self							):
								return self._config.n_layer


				@property
				def _snake_case				(						self							):
								return self._config.n_head


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = -1      ,       UpperCAmelCase_ = -1      ,       UpperCAmelCase_ = False      ,       UpperCAmelCase_ = None      ,       ):
								lowerCamelCase		       =super(UpperCAmelCase_      ,       self							).generate_dummy_inputs(
								    UpperCAmelCase_      ,       batch_size=UpperCAmelCase_      ,       seq_length=UpperCAmelCase_      ,       is_pair=UpperCAmelCase_      ,       framework=UpperCAmelCase_							)

								# We need to order the input in the way they appears in the forward()
								lowerCamelCase		       =OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}							)

								# Need to add the past_keys
								if self.use_past:
												if not is_torch_available():
																raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""							)
												else:
																import torch

																lowerCamelCase       ,     lowerCamelCase		       =common_inputs["""input_ids"""].shape
																# Not using the same length for past_key_values
																lowerCamelCase		       =seqlen + 2
																lowerCamelCase		       =(
																    batch,
																    self.num_attention_heads,
																    past_key_values_length,
																    self._config.hidden_size // self.num_attention_heads,
																)
																lowerCamelCase		       =[
																    (torch.zeros(UpperCAmelCase_							), torch.zeros(UpperCAmelCase_							)) for _ in range(self.num_layers							)
																]

								lowerCamelCase		       =common_inputs["""attention_mask"""]
								if self.use_past:
												lowerCamelCase		       =ordered_inputs["""attention_mask"""].dtype
												lowerCamelCase		       =torch.cat(
												    [ordered_inputs["""attention_mask"""], torch.ones(UpperCAmelCase_      ,       UpperCAmelCase_      ,       dtype=UpperCAmelCase_							)]      ,       dim=1							)

								return ordered_inputs


				@property
				def _snake_case				(						self							):
								return 13





