





"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCamelCase    	=      logging.get_logger(__name__)

_lowerCamelCase    	=      {
    '''google/switch-base-8''': '''https://huggingface.co/google/switch-base-8/blob/main/config.json''',
}





class      snake_case						(  _UpperCAmelCase  ):
      lowerCamelCase__         =					'switch_transformers'
      lowerCamelCase__         =					['past_key_values']
      lowerCamelCase__         =					{'hidden_size': 'd_model', 'num_attention_heads': 'num_heads', 'num_hidden_layers': 'num_layers'}

      def __init__(     self  :str		,				_lowerCamelCase  :Optional[int]=3_2_1_2_8		,				_lowerCamelCase  :List[Any]=7_6_8		,				_lowerCamelCase  :str=6_4		,				_lowerCamelCase  :Dict=2_0_4_8		,				_lowerCamelCase  :Any=6_4		,				_lowerCamelCase  :str=1_2		,				_lowerCamelCase  :Dict=3		,				_lowerCamelCase  :Tuple=1_2		,				_lowerCamelCase  :str=3		,				_lowerCamelCase  :List[Any]=1_2		,				_lowerCamelCase  :Union[str, Any]=8		,				_lowerCamelCase  :str=False		,				_lowerCamelCase  :Optional[int]=0.0_1		,				_lowerCamelCase  :Any="float32"		,				_lowerCamelCase  :str=False		,				_lowerCamelCase  :Tuple=3_2		,				_lowerCamelCase  :int=1_2_8		,				_lowerCamelCase  :Union[str, Any]=0.1		,				_lowerCamelCase  :Dict=1e-6		,				_lowerCamelCase  :List[str]=0.0_0_1		,				_lowerCamelCase  :List[Any]=0.0_0_1		,				_lowerCamelCase  :List[str]=1.0		,				_lowerCamelCase  :Any="relu"		,				_lowerCamelCase  :int=True		,				_lowerCamelCase  :Tuple=False		,				_lowerCamelCase  :Tuple=True		,				_lowerCamelCase  :Optional[int]=0		,				_lowerCamelCase  :List[str]=1		,				**_lowerCamelCase  :int		,				):
       __SCREAMING_SNAKE_CASE					:     str   				=			vocab_size
       __SCREAMING_SNAKE_CASE					:     int   				=			d_model
       __SCREAMING_SNAKE_CASE					:     List[Any]   				=			d_kv
       __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			d_ff

       __SCREAMING_SNAKE_CASE					:     Any   				=			num_sparse_encoder_layers

       __SCREAMING_SNAKE_CASE					:     Tuple   				=			num_layers
       __SCREAMING_SNAKE_CASE					:     int   				=			(
           num_decoder_layers if num_decoder_layers is not None else self.num_layers
       )  # default = symmetry
       __SCREAMING_SNAKE_CASE					:     Optional[Any]   				=			num_sparse_decoder_layers

       # This tells us, each how many encoder layer we'll have to set a sparse layer.
       if self.num_sparse_encoder_layers > 0:
        __SCREAMING_SNAKE_CASE					:     int   				=			self.num_layers // self.num_sparse_encoder_layers
       else:
        __SCREAMING_SNAKE_CASE					:     int   				=			self.num_layers  # HACK: this will create 0 sparse layers

       # This tells us, each how many encoder layer we'll have to set a sparse layer.
       if self.num_sparse_decoder_layers > 0:
        __SCREAMING_SNAKE_CASE					:     Optional[int]   				=			self.num_decoder_layers // self.num_sparse_decoder_layers
       else:
        __SCREAMING_SNAKE_CASE					:     List[Any]   				=			self.num_decoder_layers  # HACK: this will create 0 sparse layers

       __SCREAMING_SNAKE_CASE					:     Any   				=			num_heads
       __SCREAMING_SNAKE_CASE					:     Dict   				=			num_experts
       __SCREAMING_SNAKE_CASE					:     Tuple   				=			expert_capacity
       __SCREAMING_SNAKE_CASE					:     str   				=			router_bias
       __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			router_jitter_noise
       if router_dtype not in ["float32", "float16", "bfloat16"]:
        raise ValueError(f'''`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}'''		)
       __SCREAMING_SNAKE_CASE					:     str   				=			router_dtype

       __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			router_ignore_padding_tokens
       __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			relative_attention_num_buckets
       __SCREAMING_SNAKE_CASE					:     Optional[int]   				=			relative_attention_max_distance

       __SCREAMING_SNAKE_CASE					:     Dict   				=			dropout_rate
       __SCREAMING_SNAKE_CASE					:     List[Any]   				=			layer_norm_epsilon
       __SCREAMING_SNAKE_CASE					:     List[Any]   				=			initializer_factor
       __SCREAMING_SNAKE_CASE					:     Optional[int]   				=			feed_forward_proj
       __SCREAMING_SNAKE_CASE					:     Optional[Any]   				=			use_cache
       __SCREAMING_SNAKE_CASE					:     List[Any]   				=			add_router_probs

       __SCREAMING_SNAKE_CASE					:     Optional[int]   				=			router_z_loss_coef
       __SCREAMING_SNAKE_CASE					:     Tuple   				=			router_aux_loss_coef

       __SCREAMING_SNAKE_CASE					:     List[Any]   				=			self.feed_forward_proj.split('''-'''		)
       __SCREAMING_SNAKE_CASE					:     List[Any]   				=			act_info[-1]
       __SCREAMING_SNAKE_CASE					:     str   				=			act_info[0] == '''gated'''

       if len(__UpperCamelCase		) > 1 and act_info[0] != "gated" or len(__UpperCamelCase		) > 2:
        raise ValueError(
            f'''`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer.'''
            '''Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '''
            '''\'gated-gelu\' or \'relu\''''		)

       # for backwards compatibility
       if feed_forward_proj == "gated-gelu":
        __SCREAMING_SNAKE_CASE					:     str   				=			'''gelu_new'''

       super().__init__(
           pad_token_id=__UpperCamelCase		,				eos_token_id=__UpperCamelCase		,				is_encoder_decoder=__UpperCamelCase		,				**__UpperCamelCase		,				)



"""simple docstring"""



from ....configuration_utils import PretrainedConfig
from ....utils import logging


a       :						Dict       		=		logging.get_logger(__name__)

# TODO: upload to AWS
a       :						Tuple       		=		{
    '''yjernite/retribert-base-uncased''': (
        '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/config.json'''
    ),
}




class 		a_    (			_UpperCAmelCase						):
    a   :					int			     =					'retribert'







    def __init__(      self      :							Optional[Any]  ,       __UpperCamelCase      :							Optional[Any]=3_05_22  ,       __UpperCamelCase      :							int=7_68  ,       __UpperCamelCase      :							Any=8  ,       __UpperCamelCase      :							Dict=12  ,       __UpperCamelCase      :							List[Any]=30_72  ,       __UpperCamelCase      :							List[Any]="gelu"  ,       __UpperCamelCase      :							Optional[int]=0.1  ,       __UpperCamelCase      :							int=0.1  ,       __UpperCamelCase      :							Union[str, Any]=5_12  ,       __UpperCamelCase      :							Dict=2  ,       __UpperCamelCase      :							Tuple=0.0_2  ,       __UpperCamelCase      :							List[str]=1e-12  ,       __UpperCamelCase      :							int=True  ,       __UpperCamelCase      :							str=1_28  ,       __UpperCamelCase      :							Union[str, Any]=0  ,       **__UpperCamelCase      :							Any  ,       )		->List[Any]:






        '''simple docstring'''






        super().__init__(pad_token_id=__UpperCamelCase  ,       **__UpperCamelCase  )

        _UpperCAmelCase        = vocab_size
        _UpperCAmelCase        = hidden_size
        _UpperCAmelCase        = num_hidden_layers
        _UpperCAmelCase        = num_attention_heads
        _UpperCAmelCase        = hidden_act
        _UpperCAmelCase        = intermediate_size
        _UpperCAmelCase        = hidden_dropout_prob
        _UpperCAmelCase        = attention_probs_dropout_prob
        _UpperCAmelCase        = max_position_embeddings
        _UpperCAmelCase        = type_vocab_size
        _UpperCAmelCase        = initializer_range
        _UpperCAmelCase        = layer_norm_eps
        _UpperCAmelCase        = share_encoders
        _UpperCAmelCase        = projection_dim