
import warnings
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase  =logging.get_logger(__name__)




class 					_lowerCamelCase			(   UpperCamelCase_			):





     """simple docstring"""







     SCREAMING_SNAKE_CASE_		 =	['''input_ids''', '''attention_mask''']






     def __init__(				self    ,    __SCREAMING_SNAKE_CASE="</s>"    ,    __SCREAMING_SNAKE_CASE="<unk>"    ,    __SCREAMING_SNAKE_CASE="<pad>"    ,    __SCREAMING_SNAKE_CASE=1_2_5    ,    __SCREAMING_SNAKE_CASE=None    ,    **__SCREAMING_SNAKE_CASE    ,    )	-> None:




           """simple docstring"""







           if extra_ids > 0 and additional_special_tokens is None:
                 UpperCamelCase__     : Optional[int]											=					[F'''<extra_id_{i}>''' for i in range(__SCREAMING_SNAKE_CASE	)]
           elif extra_ids > 0 and additional_special_tokens is not None:
                 # Check that we have the right number of extra_id special tokens
                 UpperCamelCase__     : List[Any]											=					len(set(filter(lambda __SCREAMING_SNAKE_CASE	: bool('''extra_id''' in str(__SCREAMING_SNAKE_CASE	)	)    ,    __SCREAMING_SNAKE_CASE	)	)	)
                 if extra_tokens != extra_ids:
                       raise ValueError(
                           F'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
                           ''' provided to ByT5Tokenizer. In this case the additional_special_tokens must include the'''
                           ''' extra_ids tokens'''	)

           UpperCamelCase__     : Tuple											=					AddedToken(__SCREAMING_SNAKE_CASE    ,    lstrip=__SCREAMING_SNAKE_CASE    ,    rstrip=__SCREAMING_SNAKE_CASE	) if isinstance(__SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE	) else pad_token
           UpperCamelCase__     : Union[str, Any]											=					AddedToken(__SCREAMING_SNAKE_CASE    ,    lstrip=__SCREAMING_SNAKE_CASE    ,    rstrip=__SCREAMING_SNAKE_CASE	) if isinstance(__SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE	) else eos_token
           UpperCamelCase__     : List[str]											=					AddedToken(__SCREAMING_SNAKE_CASE    ,    lstrip=__SCREAMING_SNAKE_CASE    ,    rstrip=__SCREAMING_SNAKE_CASE	) if isinstance(__SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE	) else unk_token

           super().__init__(
               eos_token=__SCREAMING_SNAKE_CASE    ,    unk_token=__SCREAMING_SNAKE_CASE    ,    pad_token=__SCREAMING_SNAKE_CASE    ,    extra_ids=__SCREAMING_SNAKE_CASE    ,    additional_special_tokens=__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE    ,    )

           UpperCamelCase__     : List[str]											=					extra_ids

           UpperCamelCase__     : List[Any]											=					2**8  # utf is 8 bits

           # define special tokens dict
           UpperCamelCase__     : Dict[int, str]											=					{
               self.pad_token: 0,
               self.eos_token: 1,
               self.unk_token: 2,
           }
           UpperCamelCase__     : Dict											=					len(self.special_tokens_encoder	)
           UpperCamelCase__     : Dict											=					len(__SCREAMING_SNAKE_CASE	)
           for i, token in enumerate(__SCREAMING_SNAKE_CASE	):
                 UpperCamelCase__     : Tuple											=					self.vocab_size + i - n
           UpperCamelCase__     : Dict[str, int]											=					{v: k for k, v in self.special_tokens_encoder.items()}






     @property
     def  __SCREAMING_SNAKE_CASE      (				self	)	-> str:




           """simple docstring"""







           return self._utf_vocab_size + self._num_special_tokens + self._extra_ids






     def  __SCREAMING_SNAKE_CASE      (				self    ,    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE = None    ,    __SCREAMING_SNAKE_CASE = False	)	-> List[int]:




           """simple docstring"""







           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=__SCREAMING_SNAKE_CASE    ,    token_ids_a=__SCREAMING_SNAKE_CASE    ,    already_has_special_tokens=__SCREAMING_SNAKE_CASE	)

           # normal case: some special tokens
           if token_ids_a is None:
                 return ([0] * len(__SCREAMING_SNAKE_CASE	)) + [1]
           return ([0] * len(__SCREAMING_SNAKE_CASE	)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE	)) + [1]






     def  __SCREAMING_SNAKE_CASE      (				self    ,    __SCREAMING_SNAKE_CASE	)	-> List[int]:




           """simple docstring"""







           if len(__SCREAMING_SNAKE_CASE	) > 0 and token_ids[-1] == self.eos_token_id:
                 warnings.warn(
                     F'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
                     ''' eos tokens being added.'''	)
                 return token_ids
           else:
                 return token_ids + [self.eos_token_id]






     def  __SCREAMING_SNAKE_CASE      (				self    ,    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE = None	)	-> List[int]:




           """simple docstring"""







           UpperCamelCase__     : Optional[int]											=					[self.eos_token_id]

           if token_ids_a is None:
                 return len(token_ids_a + eos	) * [0]
           return len(token_ids_a + eos + token_ids_a + eos	) * [0]






     def  __SCREAMING_SNAKE_CASE      (				self    ,    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE = None	)	-> List[int]:




           """simple docstring"""







           UpperCamelCase__     : List[str]											=					self._add_eos_if_not_present(__SCREAMING_SNAKE_CASE	)
           if token_ids_a is None:
                 return token_ids_a
           else:
                 UpperCamelCase__     : Optional[Any]											=					self._add_eos_if_not_present(__SCREAMING_SNAKE_CASE	)
                 return token_ids_a + token_ids_a






     def  __SCREAMING_SNAKE_CASE      (				self    ,    __SCREAMING_SNAKE_CASE	)	-> List[str]:




           """simple docstring"""







           UpperCamelCase__     : Optional[int]											=					[chr(__SCREAMING_SNAKE_CASE	) for i in text.encode('''utf-8'''	)]
           return tokens






     def  __SCREAMING_SNAKE_CASE      (				self    ,    __SCREAMING_SNAKE_CASE	)	-> int:




           """simple docstring"""







           if token in self.special_tokens_encoder:
                 UpperCamelCase__     : Union[str, Any]											=					self.special_tokens_encoder[token]
           elif token in self.added_tokens_encoder:
                 UpperCamelCase__     : int											=					self.added_tokens_encoder[token]
           elif len(__SCREAMING_SNAKE_CASE	) != 1:
                 UpperCamelCase__     : int											=					self.unk_token_id
           else:
                 UpperCamelCase__     : Union[str, Any]											=					ord(__SCREAMING_SNAKE_CASE	) + self._num_special_tokens
           return token_id






     def  __SCREAMING_SNAKE_CASE      (				self    ,    __SCREAMING_SNAKE_CASE	)	-> Any:




           """simple docstring"""







           if index in self.special_tokens_decoder:
                 UpperCamelCase__     : List[str]											=					self.special_tokens_decoder[index]
           else:
                 UpperCamelCase__     : List[Any]											=					chr(index - self._num_special_tokens	)
           return token






     def  __SCREAMING_SNAKE_CASE      (				self    ,    __SCREAMING_SNAKE_CASE	)	-> Optional[int]:




           """simple docstring"""







           UpperCamelCase__     : List[Any]											=					B''''''
           for token in tokens:
                 if token in self.special_tokens_decoder:
                       UpperCamelCase__     : Any											=					self.special_tokens_decoder[token].encode('''utf-8'''	)
                 elif token in self.added_tokens_decoder:
                       UpperCamelCase__     : Optional[int]											=					self.special_tokens_decoder[token].encode('''utf-8'''	)
                 elif token in self.special_tokens_encoder:
                       UpperCamelCase__     : List[Any]											=					token.encode('''utf-8'''	)
                 elif token in self.added_tokens_encoder:
                       UpperCamelCase__     : Dict											=					token.encode('''utf-8'''	)
                 else:
                       UpperCamelCase__     : Optional[Any]											=					bytes([ord(__SCREAMING_SNAKE_CASE	)]	)
                 bstring += tok_string
           UpperCamelCase__     : Optional[Any]											=					bstring.decode('''utf-8'''    ,    errors='''ignore'''	)
           return string






     def  __SCREAMING_SNAKE_CASE      (				self    ,    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE = None	)	-> Tuple[str]:




           """simple docstring"""







           return ()




from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCamelCase  =logging.get_logger(__name__)

lowerCamelCase  ={
    "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/config.json",
    "distilbert-base-uncased-distilled-squad": (
        "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/config.json"
    ),
    "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/config.json",
    "distilbert-base-cased-distilled-squad": (
        "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/config.json"
    ),
    "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/config.json",
    "distilbert-base-multilingual-cased": (
        "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/config.json"
    ),
    "distilbert-base-uncased-finetuned-sst-2-english": (
        "https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english/resolve/main/config.json"
    ),
}




class 					_lowerCamelCase			(   UpperCamelCase_			):





     """simple docstring"""







     SCREAMING_SNAKE_CASE_		 =	'''distilbert'''
     SCREAMING_SNAKE_CASE_		 =	{
         '''hidden_size''': '''dim''',
         '''num_attention_heads''': '''n_heads''',
         '''num_hidden_layers''': '''n_layers''',
     }






     def __init__(				self    ,    __SCREAMING_SNAKE_CASE=3_0_5_2_2    ,    __SCREAMING_SNAKE_CASE=5_1_2    ,    __SCREAMING_SNAKE_CASE=False    ,    __SCREAMING_SNAKE_CASE=6    ,    __SCREAMING_SNAKE_CASE=1_2    ,    __SCREAMING_SNAKE_CASE=7_6_8    ,    __SCREAMING_SNAKE_CASE=4 * 7_6_8    ,    __SCREAMING_SNAKE_CASE=0.1    ,    __SCREAMING_SNAKE_CASE=0.1    ,    __SCREAMING_SNAKE_CASE="gelu"    ,    __SCREAMING_SNAKE_CASE=0.02    ,    __SCREAMING_SNAKE_CASE=0.1    ,    __SCREAMING_SNAKE_CASE=0.2    ,    __SCREAMING_SNAKE_CASE=0    ,    **__SCREAMING_SNAKE_CASE    ,    )	-> Optional[Any]:




           """simple docstring"""







           UpperCamelCase__     : List[Any]											=					vocab_size
           UpperCamelCase__     : Dict											=					max_position_embeddings
           UpperCamelCase__     : Any											=					sinusoidal_pos_embds
           UpperCamelCase__     : Dict											=					n_layers
           UpperCamelCase__     : List[Any]											=					n_heads
           UpperCamelCase__     : Dict											=					dim
           UpperCamelCase__     : Dict											=					hidden_dim
           UpperCamelCase__     : Optional[int]											=					dropout
           UpperCamelCase__     : Optional[Any]											=					attention_dropout
           UpperCamelCase__     : Tuple											=					activation
           UpperCamelCase__     : Optional[int]											=					initializer_range
           UpperCamelCase__     : Optional[int]											=					qa_dropout
           UpperCamelCase__     : str											=					seq_classif_dropout
           super().__init__(**__SCREAMING_SNAKE_CASE    ,    pad_token_id=__SCREAMING_SNAKE_CASE	)




class 					_lowerCamelCase			(   UpperCamelCase_			):





     """simple docstring"""







     @property
     def  __SCREAMING_SNAKE_CASE      (				self	)	-> Mapping[str, Mapping[int, str]]:




           """simple docstring"""







           if self.task == "multiple-choice":
                 UpperCamelCase__     : Union[str, Any]											=					{0: '''batch''', 1: '''choice''', 2: '''sequence'''}
           else:
                 UpperCamelCase__     : int											=					{0: '''batch''', 1: '''sequence'''}
           return OrderedDict(
               [
                   ('''input_ids''', dynamic_axis),
                   ('''attention_mask''', dynamic_axis),
               ]	)


