

'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCAmelCase            =		logging.get_logger(__name__)

_lowerCAmelCase            =		{
    """google/switch-base-8""": """https://huggingface.co/google/switch-base-8/blob/main/config.json""",
}






class     _SCREAMING_SNAKE_CASE       (       SCREAMING_SNAKE_CASE_      ):
 __SCREAMING_SNAKE_CASE	:Optional[Any]     			=	'switch_transformers'
 __SCREAMING_SNAKE_CASE	:Tuple     			=	['past_key_values']
 __SCREAMING_SNAKE_CASE	:List[Any]     			=	{'hidden_size': 'd_model', 'num_attention_heads': 'num_heads', 'num_hidden_layers': 'num_layers'}



 def __init__(       self						:       Any					, a__						:       Union[str, Any]=3_2128					, a__						:       Optional[int]=768					, a__						:       List[str]=64					, a__						:       List[str]=2048					, a__						:       List[str]=64					, a__						:       Tuple=12					, a__						:       int=3					, a__						:       Union[str, Any]=12					, a__						:       Tuple=3					, a__						:       Dict=12					, a__						:       Any=8					, a__						:       str=False					, a__						:       List[str]=0.01					, a__						:       Union[str, Any]="float32"					, a__						:       Optional[int]=False					, a__						:       int=32					, a__						:       str=128					, a__						:       Union[str, Any]=0.1					, a__						:       List[str]=1E-6					, a__						:       str=0.001					, a__						:       Any=0.001					, a__						:       int=1.0					, a__						:       str="relu"					, a__						:       Tuple=True					, a__						:       str=False					, a__						:       Optional[Any]=True					, a__						:       Tuple=0					, a__						:       int=1					, **a__						:       Any					, ):
      __magic_name__		  =							vocab_size
      __magic_name__		  =							d_model
      __magic_name__		  =							d_kv
      __magic_name__		  =							d_ff

      __magic_name__		  =							num_sparse_encoder_layers

      __magic_name__		  =							num_layers
      __magic_name__		  =							(
          num_decoder_layers if num_decoder_layers is not None else self.num_layers
      )  # default = symmetry
      __magic_name__		  =							num_sparse_decoder_layers

      # This tells us, each how many encoder layer we'll have to set a sparse layer.
      if self.num_sparse_encoder_layers > 0:
           __magic_name__		  =							self.num_layers // self.num_sparse_encoder_layers
      else:
           __magic_name__		  =							self.num_layers  # HACK: this will create 0 sparse layers

      # This tells us, each how many encoder layer we'll have to set a sparse layer.
      if self.num_sparse_decoder_layers > 0:
           __magic_name__		  =							self.num_decoder_layers // self.num_sparse_decoder_layers
      else:
           __magic_name__		  =							self.num_decoder_layers  # HACK: this will create 0 sparse layers

      __magic_name__		  =							num_heads
      __magic_name__		  =							num_experts
      __magic_name__		  =							expert_capacity
      __magic_name__		  =							router_bias
      __magic_name__		  =							router_jitter_noise
      if router_dtype not in ["float32", "float16", "bfloat16"]:
           raise ValueError(F'''`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}'''     )
      __magic_name__		  =							router_dtype

      __magic_name__		  =							router_ignore_padding_tokens
      __magic_name__		  =							relative_attention_num_buckets
      __magic_name__		  =							relative_attention_max_distance

      __magic_name__		  =							dropout_rate
      __magic_name__		  =							layer_norm_epsilon
      __magic_name__		  =							initializer_factor
      __magic_name__		  =							feed_forward_proj
      __magic_name__		  =							use_cache
      __magic_name__		  =							add_router_probs

      __magic_name__		  =							router_z_loss_coef
      __magic_name__		  =							router_aux_loss_coef

      __magic_name__		  =							self.feed_forward_proj.split('''-'''     )
      __magic_name__		  =							act_info[-1]
      __magic_name__		  =							act_info[0] == 'gated'

      if len(a__     ) > 1 and act_info[0] != "gated" or len(a__     ) > 2:
           raise ValueError(
               F'''`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer.'''
               '''Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '''
               '''\'gated-gelu\' or \'relu\''''     )

      # for backwards compatibility
      if feed_forward_proj == "gated-gelu":
           __magic_name__		  =							'gelu_new'

      super().__init__(
          pad_token_id=a__					, eos_token_id=a__					, is_encoder_decoder=a__					, **a__					, )











from typing import Callable, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCamelCase    :		int		      =    logging.get_logger(__name__)

_lowerCamelCase    :		Union[str, Any]		      =    {
    """microsoft/xprophetnet-large-wiki100-cased""": (
        """https://huggingface.co/microsoft/xprophetnet-large-wiki100-cased/resolve/main/config.json"""
    ),
}


class 						lowercase    (							SCREAMING_SNAKE_CASE_):
						'''simple docstring'''


						UpperCAmelCase			:		Tuple		=		'xlm-prophetnet'
						UpperCAmelCase			:		int		=		['past_key_values']
						UpperCAmelCase			:		List[Any]		=		{
						    'num_attention_heads': 'num_encoder_attention_heads',
						}







						def __init__(  self    :				List[Any]       ,	snake_case    :				Optional[float] = 0.1       ,	snake_case    :				Optional[Union[str, Callable]] = "gelu"       ,	snake_case    :				Optional[int] = 30522       ,	snake_case    :				Optional[int] = 1024       ,	snake_case    :				Optional[int] = 4096       ,	snake_case    :				Optional[int] = 12       ,	snake_case    :				Optional[int] = 16       ,	snake_case    :				Optional[int] = 4096       ,	snake_case    :				Optional[int] = 12       ,	snake_case    :				Optional[int] = 16       ,	snake_case    :				Optional[float] = 0.1       ,	snake_case    :				Optional[float] = 0.1       ,	snake_case    :				Optional[int] = 512       ,	snake_case    :				Optional[float] = 0.02       ,	snake_case    :				Optional[bool] = True       ,	snake_case    :				Optional[bool] = True       ,	snake_case    :				Optional[int] = 0       ,	snake_case    :				Optional[int] = 2       ,	snake_case    :				Optional[int] = 32       ,	snake_case    :				Optional[int] = 128       ,	snake_case    :				Optional[bool] = False       ,	snake_case    :				Optional[float] = 0.0       ,	snake_case    :				Optional[bool] = True       ,	snake_case    :				Optional[int] = 0       ,	snake_case    :				Optional[int] = 1       ,	snake_case    :				Optional[int] = 2       ,	**snake_case    :				Optional[int]       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				vocab_size
											SCREAMING_SNAKE_CASE							:     int						=				hidden_size
											SCREAMING_SNAKE_CASE							:     Dict						=				encoder_ffn_dim
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				num_encoder_layers
											SCREAMING_SNAKE_CASE							:     str						=				num_encoder_attention_heads
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				decoder_ffn_dim
											SCREAMING_SNAKE_CASE							:     Any						=				num_decoder_layers
											SCREAMING_SNAKE_CASE							:     List[str]						=				num_decoder_attention_heads
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				max_position_embeddings
											SCREAMING_SNAKE_CASE							:     Any						=				init_std  # Normal(0, this parameter)
											SCREAMING_SNAKE_CASE							:     List[str]						=				activation_function

											# parameters for xlmprophetnet
											SCREAMING_SNAKE_CASE							:     Tuple						=				ngram
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				num_buckets
											SCREAMING_SNAKE_CASE							:     Any						=				relative_max_distance
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				disable_ngram_loss
											SCREAMING_SNAKE_CASE							:     List[str]						=				eps

											# 3 Types of Dropout
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				attention_dropout
											SCREAMING_SNAKE_CASE							:     Any						=				activation_dropout
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				dropout

											SCREAMING_SNAKE_CASE							:     Tuple						=				use_cache

											super().__init__(
											    pad_token_id=snake_case       ,	bos_token_id=snake_case       ,	eos_token_id=snake_case       ,	is_encoder_decoder=snake_case       ,	add_cross_attention=snake_case       ,	decoder_start_token_id=snake_case       ,	**snake_case       ,	)







						@property
						def       lowerCamelCase_				(  self    :				Optional[Any]	):



											'''simple docstring'''


											return self.num_encoder_layers + self.num_decoder_layers







						@num_hidden_layers.setter
						def       lowerCamelCase_				(  self    :				List[Any]       ,	snake_case    :				Optional[int]	):



											'''simple docstring'''


											raise NotImplementedError(
											    'This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and'
											    ' `num_decoder_layers`.'	)