


from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE							    =							logging.get_logger(__name__)

SCREAMING_SNAKE_CASE							    =							{
    'google/switch-base-8': 'https://huggingface.co/google/switch-base-8/blob/main/config.json',
}


class     __UpperCAmelCase							(       __A							):


     """simple docstring"""
     _lowerCamelCase   =      """switch_transformers"""
     _lowerCamelCase   =      ["""past_key_values"""]
     _lowerCamelCase   =      {"""hidden_size""": """d_model""", """num_attention_heads""": """num_heads""", """num_hidden_layers""": """num_layers"""}


     def __init__(	self  ,       __A=32128  ,       __A=768  ,       __A=64  ,       __A=2048  ,       __A=64  ,       __A=12  ,       __A=3  ,       __A=12  ,       __A=3  ,       __A=12  ,       __A=8  ,       __A=False  ,       __A=0.01  ,       __A="float32"  ,       __A=False  ,       __A=32  ,       __A=128  ,       __A=0.1  ,       __A=1E-6  ,       __A=0.001  ,       __A=0.001  ,       __A=1.0  ,       __A="relu"  ,       __A=True  ,       __A=False  ,       __A=True  ,       __A=0  ,       __A=1  ,       **__A  ,       ):
        __a					=  vocab_size
        __a					=  d_model
        __a					=  d_kv
        __a					=  d_ff

        __a					=  num_sparse_encoder_layers

        __a					=  num_layers
        __a					=  (
            num_decoder_layers if num_decoder_layers is not None else self.num_layers
        )  # default = symmetry
        __a					=  num_sparse_decoder_layers

        # This tells us, each how many encoder layer we'll have to set a sparse layer.
        if self.num_sparse_encoder_layers > 0:
           __a					=  self.num_layers // self.num_sparse_encoder_layers
        else:
           __a					=  self.num_layers  # HACK: this will create 0 sparse layers

        # This tells us, each how many encoder layer we'll have to set a sparse layer.
        if self.num_sparse_decoder_layers > 0:
           __a					=  self.num_decoder_layers // self.num_sparse_decoder_layers
        else:
           __a					=  self.num_decoder_layers  # HACK: this will create 0 sparse layers

        __a					=  num_heads
        __a					=  num_experts
        __a					=  expert_capacity
        __a					=  router_bias
        __a					=  router_jitter_noise
        if router_dtype not in ["float32", "float16", "bfloat16"]:
           raise ValueError(f'''`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}'''    )
        __a					=  router_dtype

        __a					=  router_ignore_padding_tokens
        __a					=  relative_attention_num_buckets
        __a					=  relative_attention_max_distance

        __a					=  dropout_rate
        __a					=  layer_norm_epsilon
        __a					=  initializer_factor
        __a					=  feed_forward_proj
        __a					=  use_cache
        __a					=  add_router_probs

        __a					=  router_z_loss_coef
        __a					=  router_aux_loss_coef

        __a					=  self.feed_forward_proj.split("""-"""    )
        __a					=  act_info[-1]
        __a					=  act_info[0] == 'gated'

        if len(__UpperCAmelCase    ) > 1 and act_info[0] != "gated" or len(__UpperCAmelCase    ) > 2:
           raise ValueError(
               f'''`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer.'''
               """Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. """
               """\'gated-gelu\' or \'relu\'"""    )

        # for backwards compatibility
        if feed_forward_proj == "gated-gelu":
           __a					=  'gelu_new'

        super().__init__(
            pad_token_id=__UpperCAmelCase  ,       eos_token_id=__UpperCAmelCase  ,       is_encoder_decoder=__UpperCAmelCase  ,       **__UpperCAmelCase  ,       )










"""simple docstring"""





from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


if TYPE_CHECKING:
  from ... import FeatureExtractionMixin, PreTrainedTokenizerBase, TensorType


__A											=						logging.get_logger(__name__)

__A											=						{
    """microsoft/deberta-v2-xlarge""": """https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/config.json""",
    """microsoft/deberta-v2-xxlarge""": """https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/config.json""",
    """microsoft/deberta-v2-xlarge-mnli""": (
        """https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/config.json"""
    ),
    """microsoft/deberta-v2-xxlarge-mnli""": (
        """https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/config.json"""
    ),
}






class    _lowerCAmelCase  (	a			):
       """simple docstring"""




       __magic_name__	:Union[str, Any]						    =   """deberta-v2"""






       def __init__(       self       ,					__UpperCAmelCase=1_2_8_1_0_0       ,					__UpperCAmelCase=1_5_3_6       ,					__UpperCAmelCase=2_4       ,					__UpperCAmelCase=2_4       ,					__UpperCAmelCase=6_1_4_4       ,					__UpperCAmelCase="gelu"       ,					__UpperCAmelCase=0.1       ,					__UpperCAmelCase=0.1       ,					__UpperCAmelCase=5_1_2       ,					__UpperCAmelCase=0       ,					__UpperCAmelCase=0.02       ,					__UpperCAmelCase=1E-7       ,					__UpperCAmelCase=False       ,					__UpperCAmelCase=-1       ,					__UpperCAmelCase=0       ,					__UpperCAmelCase=True       ,					__UpperCAmelCase=None       ,					__UpperCAmelCase=0       ,					__UpperCAmelCase="gelu"       ,					**__UpperCAmelCase       ,					):



         '''simple docstring'''

         super().__init__(**__UpperCAmelCase )

         lowerCAmelCase__     :Optional[int]											=				hidden_size
         lowerCAmelCase__     :Optional[int]											=				num_hidden_layers
         lowerCAmelCase__     :Optional[int]											=				num_attention_heads
         lowerCAmelCase__     :int											=				intermediate_size
         lowerCAmelCase__     :Dict											=				hidden_act
         lowerCAmelCase__     :Tuple											=				hidden_dropout_prob
         lowerCAmelCase__     :Optional[int]											=				attention_probs_dropout_prob
         lowerCAmelCase__     :Optional[Any]											=				max_position_embeddings
         lowerCAmelCase__     :Optional[Any]											=				type_vocab_size
         lowerCAmelCase__     :List[Any]											=				initializer_range
         lowerCAmelCase__     :Dict											=				relative_attention
         lowerCAmelCase__     :Union[str, Any]											=				max_relative_positions
         lowerCAmelCase__     :List[str]											=				pad_token_id
         lowerCAmelCase__     :Optional[int]											=				position_biased_input

         # Backwards compatibility
         if type(__UpperCAmelCase ) == str:
           lowerCAmelCase__     :Dict											=				[x.strip() for x in pos_att_type.lower().split('|' )]

         lowerCAmelCase__     :str											=				pos_att_type
         lowerCAmelCase__     :Dict											=				vocab_size
         lowerCAmelCase__     :Tuple											=				layer_norm_eps

         lowerCAmelCase__     :Optional[Any]											=				kwargs.get('pooler_hidden_size'       ,					__UpperCAmelCase )
         lowerCAmelCase__     :Union[str, Any]											=				pooler_dropout
         lowerCAmelCase__     :int											=				pooler_hidden_act






class    _lowerCAmelCase  (	a			):
       """simple docstring"""




       @property
       def 	snake_case		(       self ):



         '''simple docstring'''

         if self.task == "multiple-choice":
           lowerCAmelCase__     :str											=				{0: 'batch', 1: 'choice', 2: 'sequence'}
         else:
           lowerCAmelCase__     :Tuple											=				{0: 'batch', 1: 'sequence'}
         if self._config.type_vocab_size > 0:
           return OrderedDict(
               [('input_ids', dynamic_axis), ('attention_mask', dynamic_axis), ('token_type_ids', dynamic_axis)] )
         else:
           return OrderedDict([('input_ids', dynamic_axis), ('attention_mask', dynamic_axis)] )






       @property
       def 	snake_case		(       self ):



         '''simple docstring'''

         return 1_2






       def 	snake_case		(       self       ,					__UpperCAmelCase       ,					__UpperCAmelCase = -1       ,					__UpperCAmelCase = -1       ,					__UpperCAmelCase = -1       ,					__UpperCAmelCase = False       ,					__UpperCAmelCase = None       ,					__UpperCAmelCase = 3       ,					__UpperCAmelCase = 4_0       ,					__UpperCAmelCase = 4_0       ,					__UpperCAmelCase = None       ,					):



         '''simple docstring'''

         lowerCAmelCase__     :Union[str, Any]											=				super().generate_dummy_inputs(preprocessor=__UpperCAmelCase       ,					framework=__UpperCAmelCase )
         if self._config.type_vocab_size == 0 and "token_type_ids" in dummy_inputs:
           del dummy_inputs["token_type_ids"]
         return dummy_inputs
