





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


lowerCAmelCase							:       int					 =	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowerCAmelCase							:       Any					 =	{
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

lowerCAmelCase							:       Optional[int]					 =	{
    "google/electra-small-generator": 512,
    "google/electra-base-generator": 512,
    "google/electra-large-generator": 512,
    "google/electra-small-discriminator": 512,
    "google/electra-base-discriminator": 512,
    "google/electra-large-discriminator": 512,
}

lowerCAmelCase							:       Dict					 =	{
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}





class 				a			(      __snake_case			):
				SCREAMING_SNAKE_CASE__     :   List[Any]		      =     VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__     :   Tuple		      =     PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__     :   str		      =     PRETRAINED_INIT_CONFIGURATION
				SCREAMING_SNAKE_CASE__     :   Optional[Any]		      =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__     :   Union[str, Any]		      =     ElectraTokenizer






				def __init__(  self			, _lowerCAmelCase=None			, _lowerCAmelCase=None			, _lowerCAmelCase=True			, _lowerCAmelCase="[UNK]"			, _lowerCAmelCase="[SEP]"			, _lowerCAmelCase="[PAD]"			, _lowerCAmelCase="[CLS]"			, _lowerCAmelCase="[MASK]"			, _lowerCAmelCase=True			, _lowerCAmelCase=None			, **_lowerCAmelCase			, ):

										"""simple docstring"""




										super().__init__(
										    snake_case_			, tokenizer_file=snake_case_			, do_lower_case=snake_case_			, unk_token=snake_case_			, sep_token=snake_case_			, pad_token=snake_case_			, cls_token=snake_case_			, mask_token=snake_case_			, tokenize_chinese_chars=snake_case_			, strip_accents=snake_case_			, **snake_case_			, )

										__SCREAMING_SNAKE_CASE:				Optional[int]           =   json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
										if (
										    normalizer_state.get('''lowercase'''			, snake_case_					) != do_lower_case
										    or normalizer_state.get('''strip_accents'''			, snake_case_					) != strip_accents
										    or normalizer_state.get('''handle_chinese_chars'''			, snake_case_					) != tokenize_chinese_chars
										):
																__SCREAMING_SNAKE_CASE:				str           =   getattr(snake_case_			, normalizer_state.pop('''type'''					)					)
																__SCREAMING_SNAKE_CASE:				List[str]           =   do_lower_case
																__SCREAMING_SNAKE_CASE:				List[Any]           =   strip_accents
																__SCREAMING_SNAKE_CASE:				Any           =   tokenize_chinese_chars
																__SCREAMING_SNAKE_CASE:				int           =   normalizer_class(**snake_case_					)

										__SCREAMING_SNAKE_CASE:				Any           =   do_lower_case






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase=None					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Any           =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

										return output






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase = None					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Dict           =   [self.sep_token_id]
										__SCREAMING_SNAKE_CASE:				Any           =   [self.cls_token_id]
										if token_ids_a is None:
																return len(cls + token_ids_a + sep					) * [0]
										return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase = None					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   self._tokenizer.model.save(snake_case_			, name=snake_case_					)
										return tuple(snake_case_					)










from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


__A							:			Optional[Any]         =			{}


try:
 if not is_sentencepiece_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __A							:			Optional[int]         =			["MLukeTokenizer"]

if TYPE_CHECKING:
 try:
  if not is_sentencepiece_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_mluke import MLukeTokenizer


else:
 import sys

 __A							:			List[str]         =			_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





