





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_           =	logging.get_logger(__name__)

UpperCAmelCase_           =	{
    "transfo-xl-wt103": "https://huggingface.co/transfo-xl-wt103/resolve/main/config.json",
}




class 			__UpperCamelCase    (							_UpperCamelCase    ):
     __A	:  Dict							  =   'transfo-xl'
     __A	:  int							  =   ['mems']
     __A	:  Optional[int]							  =   {
         'n_token': 'vocab_size',
         'hidden_size': 'd_model',
         'num_attention_heads': 'n_head',
         'num_hidden_layers': 'n_layer',
     }





     def __init__(     self				,							_UpperCamelCase=267735				,							_UpperCamelCase=[20000, 40000, 200000]				,							_UpperCamelCase=1024				,							_UpperCamelCase=1024				,							_UpperCamelCase=16				,							_UpperCamelCase=64				,							_UpperCamelCase=4096				,							_UpperCamelCase=4				,							_UpperCamelCase=False				,							_UpperCamelCase=18				,							_UpperCamelCase=1600				,							_UpperCamelCase=1000				,							_UpperCamelCase=True				,							_UpperCamelCase=True				,							_UpperCamelCase=0				,							_UpperCamelCase=-1				,							_UpperCamelCase=True				,							_UpperCamelCase=0.1				,							_UpperCamelCase=0.0				,							_UpperCamelCase=True				,							_UpperCamelCase="normal"				,							_UpperCamelCase=0.01				,							_UpperCamelCase=0.01				,							_UpperCamelCase=0.02				,							_UpperCamelCase=1e-5				,							_UpperCamelCase=0				,							**_UpperCamelCase				,							):
        _UpperCAmelCase           =   vocab_size
        _UpperCAmelCase           =   []
        self.cutoffs.extend(_UpperCAmelCase					)
        if proj_share_all_but_first:
           _UpperCAmelCase           =   [False] + [True] * len(self.cutoffs					)
        else:
           _UpperCAmelCase           =   [False] + [False] * len(self.cutoffs					)
        _UpperCAmelCase           =   d_model
        _UpperCAmelCase           =   d_embed
        _UpperCAmelCase           =   d_head
        _UpperCAmelCase           =   d_inner
        _UpperCAmelCase           =   div_val
        _UpperCAmelCase           =   pre_lnorm
        _UpperCAmelCase           =   n_layer
        _UpperCAmelCase           =   n_head
        _UpperCAmelCase           =   mem_len
        _UpperCAmelCase           =   same_length
        _UpperCAmelCase           =   attn_type
        _UpperCAmelCase           =   clamp_len
        _UpperCAmelCase           =   sample_softmax
        _UpperCAmelCase           =   adaptive
        _UpperCAmelCase           =   dropout
        _UpperCAmelCase           =   dropatt
        _UpperCAmelCase           =   untie_r
        _UpperCAmelCase           =   init
        _UpperCAmelCase           =   init_range
        _UpperCAmelCase           =   proj_init_std
        _UpperCAmelCase           =   init_std
        _UpperCAmelCase           =   layer_norm_epsilon
        super().__init__(eos_token_id=_UpperCAmelCase				,							**_UpperCAmelCase					)





     @property
     def 		UpperCamelCase(     self					):
        # Message copied from Transformer-XL documentation
        logger.info(f'''The model {self.model_type} is one of the few models that has no sequence length limit.'''					)
        return -1





     @max_position_embeddings.setter
     def 		UpperCamelCase(     self				,							_UpperCamelCase					):
        # Message copied from Transformer-XL documentation
        raise NotImplementedError(
            f'''The model {self.model_type} is one of the few models that has no sequence length limit.'''					)




'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCAmelCase		     =  logging.get_logger(__name__)

__lowerCAmelCase		     =  {
    '''s-JoL/Open-Llama-V1''': '''https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json''',
}

class       __magic_name__		(       _UpperCamelCase  ):
							lowerCAmelCase			:       str  							=   'open-llama'







							def __init__(				self   :			str      ,_UpperCAmelCase   :			int=100000      ,_UpperCAmelCase   :			List[str]=4096      ,_UpperCAmelCase   :			Dict=11008      ,_UpperCAmelCase   :			int=32      ,_UpperCAmelCase   :			Union[str, Any]=32      ,_UpperCAmelCase   :			List[str]="silu"      ,_UpperCAmelCase   :			List[Any]=2048      ,_UpperCAmelCase   :			Any=0.02      ,_UpperCAmelCase   :			int=1E-6      ,_UpperCAmelCase   :			Optional[int]=True      ,_UpperCAmelCase   :			Dict=0      ,_UpperCAmelCase   :			Optional[Any]=1      ,_UpperCAmelCase   :			Dict=2      ,_UpperCAmelCase   :			Tuple=False      ,_UpperCAmelCase   :			Dict=True      ,_UpperCAmelCase   :			Dict=0.1      ,_UpperCAmelCase   :			Optional[int]=0.1      ,_UpperCAmelCase   :			List[str]=True      ,_UpperCAmelCase   :			int=True      ,_UpperCAmelCase   :			Union[str, Any]=None      ,**_UpperCAmelCase   :			Optional[int]      ,):
									_a	: str								=				vocab_size
									_a	: str								=				max_position_embeddings
									_a	: List[str]								=				hidden_size
									_a	: Any								=				intermediate_size
									_a	: Union[str, Any]								=				num_hidden_layers
									_a	: Tuple								=				num_attention_heads
									_a	: int								=				hidden_act
									_a	: str								=				initializer_range
									_a	: Any								=				rms_norm_eps
									_a	: Dict								=				use_cache
									_a	: Optional[int]								=				kwargs.pop(
									    'use_memorry_efficient_attention'      ,_UpperCAmelCase						)
									_a	: int								=				hidden_dropout_prob
									_a	: int								=				attention_dropout_prob
									_a	: Union[str, Any]								=				use_stable_embedding
									_a	: str								=				shared_input_output_embedding
									_a	: Any								=				rope_scaling
									self._rope_scaling_validation()

									super().__init__(
									    pad_token_id=_UpperCAmelCase      ,bos_token_id=_UpperCAmelCase      ,eos_token_id=_UpperCAmelCase      ,tie_word_embeddings=_UpperCAmelCase      ,**_UpperCAmelCase      ,)







							def 				__lowercase     (				self   :			Optional[Any]						):
									if self.rope_scaling is None:
											return

									if not isinstance(self.rope_scaling      ,_UpperCAmelCase						) or len(self.rope_scaling						) != 2:
											raise ValueError(
											    '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
											    F"""got {self.rope_scaling}"""						)
									_a	: Optional[Any]								=				self.rope_scaling.get('type'      ,_UpperCAmelCase						)
									_a	: Optional[Any]								=				self.rope_scaling.get('factor'      ,_UpperCAmelCase						)
									if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
											raise ValueError(
											    F"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"""						)
									if rope_scaling_factor is None or not isinstance(_UpperCAmelCase      ,_UpperCAmelCase						) or rope_scaling_factor <= 1.0:
											raise ValueError(F"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"""						)




