






'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase :       Any 				=       logging.get_logger(__name__)

__lowerCamelCase :       int 				=       {
    '''google/switch-base-8''': '''https://huggingface.co/google/switch-base-8/blob/main/config.json''',
}







class  A_						(__SCREAMING_SNAKE_CASE				):
							"""simple docstring"""




							a__ 		=						'''switch_transformers'''
							a__ 		=						['''past_key_values''']
							a__ 		=						{'''hidden_size''': '''d_model''', '''num_attention_heads''': '''num_heads''', '''num_hidden_layers''': '''num_layers'''}

							def __init__(		self      :List[Any]							,				lowerCAmelCase__      :Dict=32_128							,				lowerCAmelCase__      :Optional[int]=768							,				lowerCAmelCase__      :List[str]=64							,				lowerCAmelCase__      :Optional[int]=2_048							,				lowerCAmelCase__      :Any=64							,				lowerCAmelCase__      :str=12							,				lowerCAmelCase__      :List[Any]=3							,				lowerCAmelCase__      :int=12							,				lowerCAmelCase__      :str=3							,				lowerCAmelCase__      :List[str]=12							,				lowerCAmelCase__      :int=8							,				lowerCAmelCase__      :Dict=False							,				lowerCAmelCase__      :Optional[int]=0.0_1							,				lowerCAmelCase__      :int="float32"							,				lowerCAmelCase__      :Any=False							,				lowerCAmelCase__      :Optional[int]=32							,				lowerCAmelCase__      :int=128							,				lowerCAmelCase__      :str=0.1							,				lowerCAmelCase__      :Dict=1E-6							,				lowerCAmelCase__      :List[str]=0.0_0_1							,				lowerCAmelCase__      :Optional[int]=0.0_0_1							,				lowerCAmelCase__      :Dict=1.0							,				lowerCAmelCase__      :Optional[int]="relu"							,				lowerCAmelCase__      :List[str]=True							,				lowerCAmelCase__      :int=False							,				lowerCAmelCase__      :Any=True							,				lowerCAmelCase__      :str=0							,				lowerCAmelCase__      :List[Any]=1							,				**lowerCAmelCase__      :Optional[int]							,				)					->	List[str]:


										'''simple docstring'''




										snake_case_					:       int =							vocab_size
										snake_case_					:       Dict =							d_model
										snake_case_					:       List[Any] =							d_kv
										snake_case_					:       Dict =							d_ff

										snake_case_					:       Optional[Any] =							num_sparse_encoder_layers

										snake_case_					:       Optional[int] =							num_layers
										snake_case_					:       Optional[Any] =							(
										    num_decoder_layers if num_decoder_layers is not None else self.num_layers
										)  # default = symmetry
										snake_case_					:       str =							num_sparse_decoder_layers

										# This tells us, each how many encoder layer we'll have to set a sparse layer.
										if self.num_sparse_encoder_layers > 0:
													snake_case_					:       Optional[Any] =							self.num_layers // self.num_sparse_encoder_layers
										else:
													snake_case_					:       int =							self.num_layers  # HACK: this will create 0 sparse layers

										# This tells us, each how many encoder layer we'll have to set a sparse layer.
										if self.num_sparse_decoder_layers > 0:
													snake_case_					:       Union[str, Any] =							self.num_decoder_layers // self.num_sparse_decoder_layers
										else:
													snake_case_					:       List[str] =							self.num_decoder_layers  # HACK: this will create 0 sparse layers

										snake_case_					:       List[Any] =							num_heads
										snake_case_					:       Optional[Any] =							num_experts
										snake_case_					:       Tuple =							expert_capacity
										snake_case_					:       Optional[int] =							router_bias
										snake_case_					:       Union[str, Any] =							router_jitter_noise
										if router_dtype not in ["float32", "float16", "bfloat16"]:
													raise ValueError(F'''`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}'''					)
										snake_case_					:       Union[str, Any] =							router_dtype

										snake_case_					:       List[Any] =							router_ignore_padding_tokens
										snake_case_					:       Any =							relative_attention_num_buckets
										snake_case_					:       str =							relative_attention_max_distance

										snake_case_					:       Any =							dropout_rate
										snake_case_					:       Tuple =							layer_norm_epsilon
										snake_case_					:       str =							initializer_factor
										snake_case_					:       Union[str, Any] =							feed_forward_proj
										snake_case_					:       List[str] =							use_cache
										snake_case_					:       Optional[int] =							add_router_probs

										snake_case_					:       Dict =							router_z_loss_coef
										snake_case_					:       Any =							router_aux_loss_coef

										snake_case_					:       Optional[Any] =							self.feed_forward_proj.split("-"					)
										snake_case_					:       Dict =							act_info[-1]
										snake_case_					:       Union[str, Any] =							act_info[0] == "gated"

										if len(__lowerCamelCase					) > 1 and act_info[0] != "gated" or len(__lowerCamelCase					) > 2:
													raise ValueError(
													    F'''`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer.'''
													    "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
													    "\'gated-gelu\' or \'relu\'"					)

										# for backwards compatibility
										if feed_forward_proj == "gated-gelu":
													snake_case_					:       Tuple =							"gelu_new"

										super().__init__(
										    pad_token_id=__lowerCamelCase							,				eos_token_id=__lowerCamelCase							,				is_encoder_decoder=__lowerCamelCase							,				**__lowerCamelCase							,				)









"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available


snake_case   					=				{'''tokenization_herbert''': ['''HerbertTokenizer''']}

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     snake_case   					=				['''HerbertTokenizerFast''']


if TYPE_CHECKING:
     from .tokenization_herbert import HerbertTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_herbert_fast import HerbertTokenizerFast

else:
     import sys

     snake_case   					=				_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





