





from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase	:       Optional[Any]    	=							logging.get_logger(__name__)

__UpperCAmelCase	:       Optional[Any]    	=							{
    "transfo-xl-wt103": "https://huggingface.co/transfo-xl-wt103/resolve/main/config.json",
}







class    __snake_case	(							__SCREAMING_SNAKE_CASE      ):





       '''simple docstring'''







       lowerCAmelCase__									=					"transfo-xl"
       lowerCAmelCase__									=					["mems"]
       lowerCAmelCase__									=					{
           "n_token": "vocab_size",
           "hidden_size": "d_model",
           "num_attention_heads": "n_head",
           "num_hidden_layers": "n_layer",
       }




       def __init__(  self			:   Union[str, Any]							,    A			:   List[Any]=267_735							,    A			:   Union[str, Any]=[20_000, 40_000, 200_000]							,    A			:   List[Any]=1_024							,    A			:   Any=1_024							,    A			:   int=16							,    A			:   List[str]=64							,    A			:   Any=4_096							,    A			:   int=4							,    A			:   Union[str, Any]=False							,    A			:   str=18							,    A			:   Union[str, Any]=1_600							,    A			:   Union[str, Any]=1_000							,    A			:   str=True							,    A			:   int=True							,    A			:   List[str]=0							,    A			:   Optional[Any]=-1							,    A			:   List[Any]=True							,    A			:   Union[str, Any]=0.1							,    A			:   int=0.0							,    A			:   Tuple=True							,    A			:   List[Any]="normal"							,    A			:   Dict=0.01							,    A			:   Optional[int]=0.01							,    A			:   Any=0.02							,    A			:   Dict=1E-5							,    A			:   Dict=0							,    **A			:   Dict							,    ):
         __snake_case:						Dict			  =						vocab_size
         __snake_case:						Tuple			  =						[]
         self.cutoffs.extend(_snake_case		)
         if proj_share_all_but_first:
           __snake_case:						Dict			  =						[False] + [True] * len(self.cutoffs		)
         else:
           __snake_case:						Union[str, Any]			  =						[False] + [False] * len(self.cutoffs		)
         __snake_case:						Optional[Any]			  =						d_model
         __snake_case:						Dict			  =						d_embed
         __snake_case:						Any			  =						d_head
         __snake_case:						str			  =						d_inner
         __snake_case:						List[str]			  =						div_val
         __snake_case:						Tuple			  =						pre_lnorm
         __snake_case:						Any			  =						n_layer
         __snake_case:						Dict			  =						n_head
         __snake_case:						List[str]			  =						mem_len
         __snake_case:						Any			  =						same_length
         __snake_case:						List[Any]			  =						attn_type
         __snake_case:						Tuple			  =						clamp_len
         __snake_case:						Optional[Any]			  =						sample_softmax
         __snake_case:						List[Any]			  =						adaptive
         __snake_case:						Tuple			  =						dropout
         __snake_case:						Optional[Any]			  =						dropatt
         __snake_case:						Any			  =						untie_r
         __snake_case:						List[str]			  =						init
         __snake_case:						List[str]			  =						init_range
         __snake_case:						Optional[int]			  =						proj_init_std
         __snake_case:						Union[str, Any]			  =						init_std
         __snake_case:						List[Any]			  =						layer_norm_epsilon
         super().__init__(eos_token_id=_snake_case							,    **_snake_case		)




       @property
       def 							UpperCAmelCase__			(  self			:   Optional[Any]		):
         # Message copied from Transformer-XL documentation
         logger.info(f'''The model {self.model_type} is one of the few models that has no sequence length limit.'''		)
         return -1






       @max_position_embeddings.setter
       def 							UpperCAmelCase__			(  self			:   Dict							,    A			:   Optional[Any]		):
         # Message copied from Transformer-XL documentation
         raise NotImplementedError(
             f'''The model {self.model_type} is one of the few models that has no sequence length limit.'''		)













from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase	:       str    	=							logging.get_logger(__name__)

__UpperCAmelCase	:       int    	=							{
    "RWKV/rwkv-4-169m-pile": "https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json",
    "RWKV/rwkv-4-430m-pile": "https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json",
    "RWKV/rwkv-4-1b5-pile": "https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json",
    "RWKV/rwkv-4-3b-pile": "https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json",
    "RWKV/rwkv-4-7b-pile": "https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json",
    "RWKV/rwkv-4-14b-pile": "https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json",
    "RWKV/rwkv-raven-1b5": "https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json",
    "RWKV/rwkv-raven-3b": "https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json",
    "RWKV/rwkv-raven-7b": "https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json",
    "RWKV/rwkv-raven-14b": "https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json",
}







class    __snake_case	(							__lowerCamelCase      ):





       '''simple docstring'''







       lowerCAmelCase__						=					"""rwkv"""
       lowerCAmelCase__						=					{"""max_position_embeddings""": """context_length"""}




       def __init__(  self			:   Dict							,    A			:   List[Any]=50_277							,    A			:   List[Any]=1_024							,    A			:   Union[str, Any]=4_096							,    A			:   Tuple=32							,    A			:   List[Any]=None							,    A			:   Tuple=None							,    A			:   Tuple=1E-5							,    A			:   int=0							,    A			:   Optional[int]=0							,    A			:   Dict=6							,    A			:   Dict=False							,    A			:   int=True							,    **A			:   List[Any]							,    ):
         __snake_case:						Tuple			  =						vocab_size
         __snake_case:						Any			  =						context_length
         __snake_case:						Dict			  =						hidden_size
         __snake_case:						Dict			  =						num_hidden_layers
         __snake_case:						Union[str, Any]			  =						attention_hidden_size if attention_hidden_size is not None else hidden_size
         __snake_case:						str			  =						intermediate_size if intermediate_size is not None else 4 * hidden_size
         __snake_case:						Any			  =						layer_norm_epsilon
         __snake_case:						int			  =						rescale_every
         __snake_case:						str			  =						use_cache

         __snake_case:						Dict			  =						bos_token_id
         __snake_case:						Union[str, Any]			  =						eos_token_id

         super().__init__(
             tie_word_embeddings=A							,    bos_token_id=A							,    eos_token_id=A							,    **A		)






