





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


__A							:			int         =			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A							:			Any         =			{
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

__A							:			Optional[int]         =			{
    "google/electra-small-generator": 512,
    "google/electra-base-generator": 512,
    "google/electra-large-generator": 512,
    "google/electra-small-discriminator": 512,
    "google/electra-base-discriminator": 512,
    "google/electra-large-discriminator": 512,
}

__A							:			Dict         =			{
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}







class      lowerCamelCase(						__snake_case	):
     '''simple docstring'''


     __magic_name__   =			VOCAB_FILES_NAMES
     __magic_name__   =			PRETRAINED_VOCAB_FILES_MAP
     __magic_name__   =			PRETRAINED_INIT_CONFIGURATION
     __magic_name__   =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__   =			ElectraTokenizer




     def __init__(    self				,     snake_case_=None				,     snake_case_=None				,     snake_case_=True				,     snake_case_="[UNK]"				,     snake_case_="[SEP]"				,     snake_case_="[PAD]"				,     snake_case_="[CLS]"				,     snake_case_="[MASK]"				,     snake_case_=True				,     snake_case_=None				,     **snake_case_				,     ):
         super().__init__(
             snake_case_				,     tokenizer_file=snake_case_				,     do_lower_case=snake_case_				,     unk_token=snake_case_				,     sep_token=snake_case_				,     pad_token=snake_case_				,     cls_token=snake_case_				,     mask_token=snake_case_				,     tokenize_chinese_chars=snake_case_				,     strip_accents=snake_case_				,     **snake_case_				,     )

         _A     =							json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
         if (
             normalizer_state.get('lowercase'				,     snake_case_							) != do_lower_case
             or normalizer_state.get('strip_accents'				,     snake_case_							) != strip_accents
             or normalizer_state.get('handle_chinese_chars'				,     snake_case_							) != tokenize_chinese_chars
         ):
             _A     =							getattr(snake_case_				,     normalizer_state.pop('type'							)							)
             _A     =							do_lower_case
             _A     =							strip_accents
             _A     =							tokenize_chinese_chars
             _A     =							normalizer_class(**snake_case_							)

         _A     =							do_lower_case




     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_=None							):
         _A     =							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

         return output




     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_ = None							):
         _A     =							[self.sep_token_id]
         _A     =							[self.cls_token_id]
         if token_ids_a is None:
             return len(cls + token_ids_a + sep							) * [0]
         return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]




     def        lowerCAmelCase__		(    self				,     snake_case_				,     snake_case_ = None							):
         _A     =							self._tokenizer.model.save(snake_case_				,     name=snake_case_							)
         return tuple(snake_case_							)












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


__A							:			Optional[Any]         =			{}


try:
 if not is_sentencepiece_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __A							:			Optional[int]         =			["MLukeTokenizer"]

if TYPE_CHECKING:
 try:
  if not is_sentencepiece_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_mluke import MLukeTokenizer


else:
 import sys

 __A							:			List[str]         =			_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





