





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCAmelCase        =				logging.get_logger(__name__)

_UpperCAmelCase        =				{
    'google/switch-base-8': 'https://huggingface.co/google/switch-base-8/blob/main/config.json',
}



class 							_UpperCamelCase					(	lowerCAmelCase_     ):
       _UpperCamelCase				:					Any         =   '''switch_transformers'''
       _UpperCamelCase				:					List[str]         =   ['''past_key_values''']
       _UpperCamelCase				:					Union[str, Any]         =   {'''hidden_size''': '''d_model''', '''num_attention_heads''': '''num_heads''', '''num_hidden_layers''': '''num_layers'''}

       def __init__(		self: Dict					,   _SCREAMING_SNAKE_CASE: Any=32128					,   _SCREAMING_SNAKE_CASE: List[str]=768					,   _SCREAMING_SNAKE_CASE: str=64					,   _SCREAMING_SNAKE_CASE: List[str]=2048					,   _SCREAMING_SNAKE_CASE: Tuple=64					,   _SCREAMING_SNAKE_CASE: Dict=12					,   _SCREAMING_SNAKE_CASE: Any=3					,   _SCREAMING_SNAKE_CASE: List[str]=12					,   _SCREAMING_SNAKE_CASE: Optional[Any]=3					,   _SCREAMING_SNAKE_CASE: Optional[Any]=12					,   _SCREAMING_SNAKE_CASE: Any=8					,   _SCREAMING_SNAKE_CASE: str=False					,   _SCREAMING_SNAKE_CASE: Any=0.01					,   _SCREAMING_SNAKE_CASE: int="float32"					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=False					,   _SCREAMING_SNAKE_CASE: Optional[int]=32					,   _SCREAMING_SNAKE_CASE: Optional[int]=128					,   _SCREAMING_SNAKE_CASE: Dict=0.1					,   _SCREAMING_SNAKE_CASE: Any=1e-6					,   _SCREAMING_SNAKE_CASE: str=0.0_01					,   _SCREAMING_SNAKE_CASE: Any=0.0_01					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=1.0					,   _SCREAMING_SNAKE_CASE: Tuple="relu"					,   _SCREAMING_SNAKE_CASE: Optional[int]=True					,   _SCREAMING_SNAKE_CASE: List[str]=False					,   _SCREAMING_SNAKE_CASE: Optional[int]=True					,   _SCREAMING_SNAKE_CASE: str=0					,   _SCREAMING_SNAKE_CASE: Any=1					,   **_SCREAMING_SNAKE_CASE: Any					,   )       -> Union[str, Any]:




              """simple docstring"""


              UpperCamelCase_          =  vocab_size
              UpperCamelCase_          =  d_model
              UpperCamelCase_          =  d_kv
              UpperCamelCase_          =  d_ff

              UpperCamelCase_          =  num_sparse_encoder_layers

              UpperCamelCase_          =  num_layers
              UpperCamelCase_          =  (
                  num_decoder_layers if num_decoder_layers is not None else self.num_layers
              )  # default = symmetry
              UpperCamelCase_          =  num_sparse_decoder_layers

              # This tells us, each how many encoder layer we'll have to set a sparse layer.
              if self.num_sparse_encoder_layers > 0:
                     UpperCamelCase_          =  self.num_layers // self.num_sparse_encoder_layers
              else:
                     UpperCamelCase_          =  self.num_layers  # HACK: this will create 0 sparse layers

              # This tells us, each how many encoder layer we'll have to set a sparse layer.
              if self.num_sparse_decoder_layers > 0:
                     UpperCamelCase_          =  self.num_decoder_layers // self.num_sparse_decoder_layers
              else:
                     UpperCamelCase_          =  self.num_decoder_layers  # HACK: this will create 0 sparse layers

              UpperCamelCase_          =  num_heads
              UpperCamelCase_          =  num_experts
              UpperCamelCase_          =  expert_capacity
              UpperCamelCase_          =  router_bias
              UpperCamelCase_          =  router_jitter_noise
              if router_dtype not in ["float32", "float16", "bfloat16"]:
                     raise ValueError(f'''`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}'''		)
              UpperCamelCase_          =  router_dtype

              UpperCamelCase_          =  router_ignore_padding_tokens
              UpperCamelCase_          =  relative_attention_num_buckets
              UpperCamelCase_          =  relative_attention_max_distance

              UpperCamelCase_          =  dropout_rate
              UpperCamelCase_          =  layer_norm_epsilon
              UpperCamelCase_          =  initializer_factor
              UpperCamelCase_          =  feed_forward_proj
              UpperCamelCase_          =  use_cache
              UpperCamelCase_          =  add_router_probs

              UpperCamelCase_          =  router_z_loss_coef
              UpperCamelCase_          =  router_aux_loss_coef

              UpperCamelCase_          =  self.feed_forward_proj.split("-"		)
              UpperCamelCase_          =  act_info[-1]
              UpperCamelCase_          =  act_info[0] == "gated"

              if len(_SCREAMING_SNAKE_CASE		) > 1 and act_info[0] != "gated" or len(_SCREAMING_SNAKE_CASE		) > 2:
                     raise ValueError(
                         f'''`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer.'''
                         "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
                         "'gated-gelu' or 'relu'"		)

              # for backwards compatibility
              if feed_forward_proj == "gated-gelu":
                     UpperCamelCase_          =  "gelu_new"

              super().__init__(
                  pad_token_id=_SCREAMING_SNAKE_CASE					,   eos_token_id=_SCREAMING_SNAKE_CASE					,   is_encoder_decoder=_SCREAMING_SNAKE_CASE					,   **_SCREAMING_SNAKE_CASE					,   )








from typing import List, Optional

from tokenizers import ByteLevelBPETokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_blenderbot_small import BlenderbotSmallTokenizer


_UpperCAmelCase        =				logging.get_logger(__name__)

_UpperCAmelCase        =				{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

_UpperCAmelCase        =				{
    'vocab_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json'
    },
    'merges_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt'
    },
    'tokenizer_config_file': {
        'facebook/blenderbot_small-90M': (
            'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json'
        )
    },
}

_UpperCAmelCase        =				{
    'facebook/blenderbot_small-90M': 5_1_2,
}



class 							_UpperCamelCase					(	lowerCAmelCase_     ):
       _UpperCamelCase				:					Optional[Any]         =   VOCAB_FILES_NAMES
       _UpperCamelCase				:					str         =   PRETRAINED_VOCAB_FILES_MAP
       _UpperCamelCase				:					List[Any]         =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _UpperCamelCase				:					str         =   BlenderbotSmallTokenizer

       def __init__(		self: Dict					,   _SCREAMING_SNAKE_CASE: Optional[int]=None					,   _SCREAMING_SNAKE_CASE: List[str]=None					,   _SCREAMING_SNAKE_CASE: Optional[int]="<|endoftext|>"					,   _SCREAMING_SNAKE_CASE: List[Any]="<|endoftext|>"					,   _SCREAMING_SNAKE_CASE: List[str]="<|endoftext|>"					,   _SCREAMING_SNAKE_CASE: Optional[Any]=False					,   _SCREAMING_SNAKE_CASE: int=True					,   **_SCREAMING_SNAKE_CASE: Optional[int]					,   )       -> int:




              """simple docstring"""


              super().__init__(
                  ByteLevelBPETokenizer(
                      vocab=_SCREAMING_SNAKE_CASE					,   merges=_SCREAMING_SNAKE_CASE					,   add_prefix_space=_SCREAMING_SNAKE_CASE					,   trim_offsets=_SCREAMING_SNAKE_CASE					,   )					,   bos_token=_SCREAMING_SNAKE_CASE					,   eos_token=_SCREAMING_SNAKE_CASE					,   unk_token=_SCREAMING_SNAKE_CASE					,   **_SCREAMING_SNAKE_CASE					,   )
              UpperCamelCase_          =  add_prefix_space

       def lowercase (		self: Any					,   _SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: Any=None		)       -> List[str]:




              """simple docstring"""


              UpperCamelCase_          =  [self.bos_token_id] + token_ids_a + [self.eos_token_id]
              if token_ids_a is None:
                     return output

              return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



       def lowercase (		self: int					,   _SCREAMING_SNAKE_CASE: List[int]					,   _SCREAMING_SNAKE_CASE: Optional[List[int]] = None		)       -> List[int]:




              """simple docstring"""


              UpperCamelCase_          =  [self.sep_token_id]
              UpperCamelCase_          =  [self.cls_token_id]

              if token_ids_a is None:
                     return len(cls + token_ids_a + sep		) * [0]
              return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]

