





import random
from typing import List, Optional, Union

import numpy as np
import torch

from ..state import AcceleratorState
from .constants import CUDA_DISTRIBUTED_TYPES
from .dataclasses import DistributedType, RNGType
from .imports import is_npu_available, is_tpu_available, is_xpu_available


if is_tpu_available(check_device=False):
							import torch_xla.core.xla_model as xm





def    _lowercase    (  _UpperCAmelCase	,					_UpperCAmelCase = False )  ->     str:
				if device_specific:
								seed += AcceleratorState().process_index
				random.seed(_UpperCAmelCase )
				np.random.seed(_UpperCAmelCase )
				torch.manual_seed(_UpperCAmelCase )
				if is_xpu_available():
								torch.xpu.manual_seed_all(_UpperCAmelCase )
				elif is_npu_available():
								torch.npu.manual_seed_all(_UpperCAmelCase )
				else:
								torch.cuda.manual_seed_all(_UpperCAmelCase )
				# ^^ safe to call this function even if cuda is not available
				if is_tpu_available():
								xm.set_rng_state(_UpperCAmelCase )





def    _lowercase    (  _UpperCAmelCase = None	,					_UpperCAmelCase = None )  ->     Dict:
				# Get the proper rng state
				if rng_type == RNGType.TORCH:
								lowerCamelCase		       =torch.get_rng_state()
				elif rng_type == RNGType.CUDA:
								lowerCamelCase		       =torch.cuda.get_rng_state()
				elif rng_type == RNGType.XLA:
								assert is_tpu_available(), "Can't synchronize XLA seeds on an environment without TPUs."
								lowerCamelCase		       =torch.tensor(xm.get_rng_state() )
				elif rng_type == RNGType.NPU:
								assert is_npu_available(), "Can't synchronize NPU seeds on an environment without NPUs."
								lowerCamelCase		       =torch.npu.get_rng_state()
				elif rng_type == RNGType.XPU:
								assert is_xpu_available(), "Can't synchronize XPU seeds on an environment without XPUs."
								lowerCamelCase		       =torch.xpu.get_rng_state()
				elif rng_type == RNGType.GENERATOR:
								assert generator is not None, "Need a generator to synchronize its seed."
								lowerCamelCase		       =generator.get_state()

				# Broadcast the rng state from device 0 to other devices
				lowerCamelCase		       =AcceleratorState()
				if state.distributed_type == DistributedType.TPU:
								lowerCamelCase		       =rng_state.to(xm.xla_device() )
								xm.collective_broadcast([rng_state] )
								xm.mark_step()
								lowerCamelCase		       =rng_state.cpu()
				elif (
				    state.distributed_type in CUDA_DISTRIBUTED_TYPES
				    or state.distributed_type == DistributedType.MULTI_NPU
				    or state.distributed_type == DistributedType.MULTI_XPU
				):
								lowerCamelCase		       =rng_state.to(state.device )
								torch.distributed.broadcast(_UpperCAmelCase	,					0 )
								lowerCamelCase		       =rng_state.cpu()
				elif state.distributed_type == DistributedType.MULTI_CPU:
								torch.distributed.broadcast(_UpperCAmelCase	,					0 )

				# Set the broadcast rng state
				if rng_type == RNGType.TORCH:
								torch.set_rng_state(_UpperCAmelCase )
				elif rng_type == RNGType.CUDA:
								torch.cuda.set_rng_state(_UpperCAmelCase )
				elif rng_type == RNGType.NPU:
								torch.npu.set_rng_state(_UpperCAmelCase )
				elif rng_type == RNGType.XPU:
								torch.xpu.set_rng_state(_UpperCAmelCase )
				elif rng_type == RNGType.XLA:
								xm.set_rng_state(rng_state.item() )
				elif rng_type == RNGType.GENERATOR:
								generator.set_state(_UpperCAmelCase )





def    _lowercase    (  _UpperCAmelCase	,					_UpperCAmelCase = None )  ->     Optional[Any]:
				for rng_type in rng_types:
								synchronize_rng_state(RNGType(_UpperCAmelCase )	,					generator=_UpperCAmelCase )












from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


UpperCAmelCase__  :	str     =logging.get_logger(__name__)

UpperCAmelCase__  :	Optional[int]     ={
    '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/config.json''',
    '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/config.json''',
    '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/config.json''',
    '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/config.json''',
    '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/config.json''',
}

class 					__A   (  a			):
				__A            = """gpt2"""
				__A            = ["""past_key_values"""]
				__A            = {
				    """hidden_size""": """n_embd""",
				    """max_position_embeddings""": """n_positions""",
				    """num_attention_heads""": """n_head""",
				    """num_hidden_layers""": """n_layer""",
				}


				def __init__(						self      ,       UpperCAmelCase_=50257      ,       UpperCAmelCase_=1024      ,       UpperCAmelCase_=768      ,       UpperCAmelCase_=12      ,       UpperCAmelCase_=12      ,       UpperCAmelCase_=None      ,       UpperCAmelCase_="gelu_new"      ,       UpperCAmelCase_=0.1      ,       UpperCAmelCase_=0.1      ,       UpperCAmelCase_=0.1      ,       UpperCAmelCase_=1E-5      ,       UpperCAmelCase_=0.0_2      ,       UpperCAmelCase_="cls_index"      ,       UpperCAmelCase_=True      ,       UpperCAmelCase_=None      ,       UpperCAmelCase_=True      ,       UpperCAmelCase_=0.1      ,       UpperCAmelCase_=True      ,       UpperCAmelCase_=True      ,       UpperCAmelCase_=50256      ,       UpperCAmelCase_=50256      ,       UpperCAmelCase_=False      ,       UpperCAmelCase_=False      ,       **UpperCAmelCase_      ,       ):
								lowerCamelCase		       =vocab_size
								lowerCamelCase		       =n_positions
								lowerCamelCase		       =n_embd
								lowerCamelCase		       =n_layer
								lowerCamelCase		       =n_head
								lowerCamelCase		       =n_inner
								lowerCamelCase		       =activation_function
								lowerCamelCase		       =resid_pdrop
								lowerCamelCase		       =embd_pdrop
								lowerCamelCase		       =attn_pdrop
								lowerCamelCase		       =layer_norm_epsilon
								lowerCamelCase		       =initializer_range
								lowerCamelCase		       =summary_type
								lowerCamelCase		       =summary_use_proj
								lowerCamelCase		       =summary_activation
								lowerCamelCase		       =summary_first_dropout
								lowerCamelCase		       =summary_proj_to_labels
								lowerCamelCase		       =scale_attn_weights
								lowerCamelCase		       =use_cache
								lowerCamelCase		       =scale_attn_by_inverse_layer_idx
								lowerCamelCase		       =reorder_and_upcast_attn

								lowerCamelCase		       =bos_token_id
								lowerCamelCase		       =eos_token_id

								super().__init__(bos_token_id=UpperCAmelCase_      ,       eos_token_id=UpperCAmelCase_      ,       **UpperCAmelCase_							)




class 					__A   (  a			):


				def __init__(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = "default"      ,       UpperCAmelCase_ = None      ,       UpperCAmelCase_ = False      ,       ):
								super().__init__(UpperCAmelCase_      ,       task=UpperCAmelCase_      ,       patching_specs=UpperCAmelCase_      ,       use_past=UpperCAmelCase_							)
								if not getattr(self._config      ,       """pad_token_id"""      ,       UpperCAmelCase_							):
												# TODO: how to do that better?
												lowerCamelCase		       =0


				@property
				def _snake_case				(						self							):
								lowerCamelCase		       =OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}							)
								if self.use_past:
												self.fill_with_past_key_values_(UpperCAmelCase_      ,       direction="""inputs"""							)
												lowerCamelCase		       ={0: """batch""", 1: """past_sequence + sequence"""}
								else:
												lowerCamelCase		       ={0: """batch""", 1: """sequence"""}

								return common_inputs


				@property
				def _snake_case				(						self							):
								return self._config.n_layer


				@property
				def _snake_case				(						self							):
								return self._config.n_head


				def _snake_case				(						self      ,       UpperCAmelCase_      ,       UpperCAmelCase_ = -1      ,       UpperCAmelCase_ = -1      ,       UpperCAmelCase_ = False      ,       UpperCAmelCase_ = None      ,       ):
								lowerCamelCase		       =super(UpperCAmelCase_      ,       self							).generate_dummy_inputs(
								    UpperCAmelCase_      ,       batch_size=UpperCAmelCase_      ,       seq_length=UpperCAmelCase_      ,       is_pair=UpperCAmelCase_      ,       framework=UpperCAmelCase_							)

								# We need to order the input in the way they appears in the forward()
								lowerCamelCase		       =OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}							)

								# Need to add the past_keys
								if self.use_past:
												if not is_torch_available():
																raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""							)
												else:
																import torch

																lowerCamelCase       ,     lowerCamelCase		       =common_inputs["""input_ids"""].shape
																# Not using the same length for past_key_values
																lowerCamelCase		       =seqlen + 2
																lowerCamelCase		       =(
																    batch,
																    self.num_attention_heads,
																    past_key_values_length,
																    self._config.hidden_size // self.num_attention_heads,
																)
																lowerCamelCase		       =[
																    (torch.zeros(UpperCAmelCase_							), torch.zeros(UpperCAmelCase_							)) for _ in range(self.num_layers							)
																]

								lowerCamelCase		       =common_inputs["""attention_mask"""]
								if self.use_past:
												lowerCamelCase		       =ordered_inputs["""attention_mask"""].dtype
												lowerCamelCase		       =torch.cat(
												    [ordered_inputs["""attention_mask"""], torch.ones(UpperCAmelCase_      ,       UpperCAmelCase_      ,       dtype=UpperCAmelCase_							)]      ,       dim=1							)

								return ordered_inputs


				@property
				def _snake_case				(						self							):
								return 13





