

"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case_   :   int        =       logging.get_logger(__name__)

snake_case_   :   Any        =       {
    """tanreinama/GPTSAN-2.8B-spout_is_uniform""": (
        """https://huggingface.co/tanreinama/GPTSAN-2.8B-spout_is_uniform/resolve/main/config.json"""
    ),
}







class       snake_case__  ( lowerCAmelCase_     ):
			SCREAMING_SNAKE_CASE__     				=						'''gptsan-japanese'''
			SCREAMING_SNAKE_CASE__     				=						[
			    '''past_key_values''',
			]
			SCREAMING_SNAKE_CASE__     				=						{
			    '''hidden_size''': '''d_model''',
			    '''num_attention_heads''': '''num_heads''',
			    '''num_hidden_layers''': '''num_layers''',
			}







			def __init__(			self			: List[str]			,  lowercase			: Dict=3_60_00			,  lowercase			: Tuple=12_80			,  lowercase			: Any=10_24			,  lowercase			: Optional[Any]=81_92			,  lowercase			: Optional[int]=40_96			,  lowercase			: Optional[Any]=1_28			,  lowercase			: Optional[int]=10			,  lowercase			: Any=0			,  lowercase			: Tuple=16			,  lowercase			: Any=16			,  lowercase			: Union[str, Any]=1_28			,  lowercase			: List[str]=0.0			,  lowercase			: Dict=1E-5			,  lowercase			: Tuple=False			,  lowercase			: int=0.0			,  lowercase			: Optional[Any]="float32"			,  lowercase			: Any=False			,  lowercase			: Tuple=False			,  lowercase			: Any=False			,  lowercase			: int=0.0_0_2			,  lowercase			: Any=False			,  lowercase			: Optional[int]=True			,  lowercase			: List[Any]=3_59_98			,  lowercase			: str=3_59_95			,  lowercase			: int=3_59_99			,  **lowercase			: str			,  ):





								'''simple docstring'''



								UpperCAmelCase			:   Tuple      =				vocab_size
								UpperCAmelCase			:   Tuple      =				max_position_embeddings
								UpperCAmelCase			:   Any      =				d_model
								UpperCAmelCase			:   List[str]      =				d_ff
								UpperCAmelCase			:   int      =				d_ext
								UpperCAmelCase			:   Optional[int]      =				d_spout
								UpperCAmelCase			:   int      =				num_switch_layers
								UpperCAmelCase			:   Tuple      =				num_ext_layers
								UpperCAmelCase			:   Optional[Any]      =				num_switch_layers + num_ext_layers
								UpperCAmelCase			:   Optional[int]      =				num_heads
								UpperCAmelCase			:   Tuple      =				num_experts
								UpperCAmelCase			:   Union[str, Any]      =				expert_capacity
								UpperCAmelCase			:   Tuple      =				dropout_rate
								UpperCAmelCase			:   List[Any]      =				layer_norm_epsilon
								UpperCAmelCase			:   Union[str, Any]      =				router_bias
								UpperCAmelCase			:   Optional[int]      =				router_jitter_noise
								UpperCAmelCase			:   Union[str, Any]      =				router_dtype
								UpperCAmelCase			:   Union[str, Any]      =				router_ignore_padding_tokens
								UpperCAmelCase			:   List[Any]      =				output_hidden_states
								UpperCAmelCase			:   Union[str, Any]      =				output_attentions
								UpperCAmelCase			:   Tuple      =				initializer_factor
								UpperCAmelCase			:   str      =				output_router_logits
								UpperCAmelCase			:   Any      =				use_cache

								super().__init__(
								    separator_token_id=lowercase			,  pad_token_id=lowercase			,  eos_token_id=lowercase			,  **lowercase			,  )





"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case_   :   List[Any]        =       logging.get_logger(__name__)

snake_case_   :   List[Any]        =       {
    """RWKV/rwkv-4-169m-pile""": """https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-430m-pile""": """https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-1b5-pile""": """https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-3b-pile""": """https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-7b-pile""": """https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json""",
    """RWKV/rwkv-4-14b-pile""": """https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json""",
    """RWKV/rwkv-raven-1b5""": """https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json""",
    """RWKV/rwkv-raven-3b""": """https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json""",
    """RWKV/rwkv-raven-7b""": """https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json""",
    """RWKV/rwkv-raven-14b""": """https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json""",
}







class       snake_case__  ( lowerCAmelCase_     ):
			SCREAMING_SNAKE_CASE__     				=						'''rwkv'''
			SCREAMING_SNAKE_CASE__     				=						{'''max_position_embeddings''': '''context_length'''}







			def __init__(			self			: int			,  lowercase			: Optional[Any]=5_02_77			,  lowercase			: Union[str, Any]=10_24			,  lowercase			: Union[str, Any]=40_96			,  lowercase			: List[str]=32			,  lowercase			: str=None			,  lowercase			: Tuple=None			,  lowercase			: Dict=1E-5			,  lowercase			: Any=0			,  lowercase			: List[Any]=0			,  lowercase			: int=6			,  lowercase			: Dict=False			,  lowercase			: Dict=True			,  **lowercase			: Optional[Any]			,  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[int]      =				vocab_size
								UpperCAmelCase			:   List[str]      =				context_length
								UpperCAmelCase			:   Optional[Any]      =				hidden_size
								UpperCAmelCase			:   List[Any]      =				num_hidden_layers
								UpperCAmelCase			:   Union[str, Any]      =				attention_hidden_size if attention_hidden_size is not None else hidden_size
								UpperCAmelCase			:   List[Any]      =				intermediate_size if intermediate_size is not None else 4 * hidden_size
								UpperCAmelCase			:   Tuple      =				layer_norm_epsilon
								UpperCAmelCase			:   Optional[int]      =				rescale_every
								UpperCAmelCase			:   Tuple      =				use_cache

								UpperCAmelCase			:   Union[str, Any]      =				bos_token_id
								UpperCAmelCase			:   Optional[int]      =				eos_token_id

								super().__init__(
								    tie_word_embeddings=lowercase			,  bos_token_id=lowercase			,  eos_token_id=lowercase			,  **lowercase  )


