


'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowercase     :						List[str]	=     logging.get_logger(__name__)

__lowercase     :						Tuple	=     {
    '''google/switch-base-8''': '''https://huggingface.co/google/switch-base-8/blob/main/config.json''',
}





class 							__UpperCamelCase    (		SCREAMING_SNAKE_CASE_ ):
	A_           =   "switch_transformers"
	A_           =   ["past_key_values"]
	A_           =   {"hidden_size": "d_model", "num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}




	def __init__(       self						,    __a=3_2128						,    __a=768						,    __a=64						,    __a=2048						,    __a=64						,    __a=12						,    __a=3						,    __a=12						,    __a=3						,    __a=12						,    __a=8						,    __a=False						,    __a=0.01						,    __a="float32"						,    __a=False						,    __a=32						,    __a=128						,    __a=0.1						,    __a=1E-6						,    __a=0.001						,    __a=0.001						,    __a=1.0						,    __a="relu"						,    __a=True						,    __a=False						,    __a=True						,    __a=0						,    __a=1						,    **__a						,    ):

						'''simple docstring'''





						__a      :	Tuple    =  vocab_size
						__a      :	List[str]    =  d_model
						__a      :	List[Any]    =  d_kv
						__a      :	str    =  d_ff

						__a      :	str    =  num_sparse_encoder_layers

						__a      :	Any    =  num_layers
						__a      :	List[str]    =  (
						    num_decoder_layers if num_decoder_layers is not None else self.num_layers
						)  # default = symmetry
						__a      :	Any    =  num_sparse_decoder_layers

						# This tells us, each how many encoder layer we'll have to set a sparse layer.
						if self.num_sparse_encoder_layers > 0:
											__a      :	List[str]    =  self.num_layers // self.num_sparse_encoder_layers
						else:
											__a      :	List[Any]    =  self.num_layers  # HACK: this will create 0 sparse layers

						# This tells us, each how many encoder layer we'll have to set a sparse layer.
						if self.num_sparse_decoder_layers > 0:
											__a      :	Union[str, Any]    =  self.num_decoder_layers // self.num_sparse_decoder_layers
						else:
											__a      :	Optional[Any]    =  self.num_decoder_layers  # HACK: this will create 0 sparse layers

						__a      :	Tuple    =  num_heads
						__a      :	int    =  num_experts
						__a      :	Tuple    =  expert_capacity
						__a      :	List[str]    =  router_bias
						__a      :	Tuple    =  router_jitter_noise
						if router_dtype not in ["float32", "float16", "bfloat16"]:
											raise ValueError(f"""`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}"""							)
						__a      :	Optional[Any]    =  router_dtype

						__a      :	Dict    =  router_ignore_padding_tokens
						__a      :	Dict    =  relative_attention_num_buckets
						__a      :	Union[str, Any]    =  relative_attention_max_distance

						__a      :	Dict    =  dropout_rate
						__a      :	int    =  layer_norm_epsilon
						__a      :	str    =  initializer_factor
						__a      :	Optional[int]    =  feed_forward_proj
						__a      :	List[str]    =  use_cache
						__a      :	List[str]    =  add_router_probs

						__a      :	Optional[int]    =  router_z_loss_coef
						__a      :	Optional[int]    =  router_aux_loss_coef

						__a      :	List[Any]    =  self.feed_forward_proj.split('-'							)
						__a      :	str    =  act_info[-1]
						__a      :	Optional[Any]    =  act_info[0] == 'gated'

						if len(__a							) > 1 and act_info[0] != "gated" or len(__a							) > 2:
											raise ValueError(
											    f"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
											    'Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '
											    '\'gated-gelu\' or \'relu\''							)

						# for backwards compatibility
						if feed_forward_proj == "gated-gelu":
											__a      :	int    =  'gelu_new'

						super().__init__(
						    pad_token_id=__a						,    eos_token_id=__a						,    is_encoder_decoder=__a						,    **__a						,    )







'''simple docstring'''



from __future__ import annotations

from typing import Dict

from ...configuration_utils import PretrainedConfig


__lowercase     :						List[Any]	=     {
    'susnato/ernie-m-base_pytorch': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/config.json',
    'susnato/ernie-m-large_pytorch': 'https://huggingface.co/susnato/ernie-m-large_pytorch/blob/main/config.json',
}





class 							__UpperCamelCase    (		lowerCAmelCase_ ):
	A_           =   "ernie_m"
	A_           =   {"dropout": "classifier_dropout", "num_classes": "num_labels"}




	def __init__(       self						,    __a = 25_0002						,    __a = 768						,    __a = 12						,    __a = 12						,    __a = 3072						,    __a = "gelu"						,    __a = 0.1						,    __a = 0.1						,    __a = 514						,    __a = 0.02						,    __a = 1						,    __a = 1E-0_5						,    __a=None						,    __a=False						,    __a=0.0						,    **__a						,    ):

						'''simple docstring'''





						super().__init__(pad_token_id=__a						,    **__a							)
						__a      :	int    =  vocab_size
						__a      :	Dict    =  hidden_size
						__a      :	str    =  num_hidden_layers
						__a      :	Dict    =  num_attention_heads
						__a      :	List[str]    =  intermediate_size
						__a      :	Union[str, Any]    =  hidden_act
						__a      :	List[Any]    =  hidden_dropout_prob
						__a      :	str    =  attention_probs_dropout_prob
						__a      :	Any    =  max_position_embeddings
						__a      :	int    =  initializer_range
						__a      :	Dict    =  layer_norm_eps
						__a      :	int    =  classifier_dropout
						__a      :	Dict    =  is_decoder
						__a      :	int    =  act_dropout



