





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


UpperCAmelCase_      							=     {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase_      							=     {
    """vocab_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-generator""": """https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt""",
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-generator""": (
            """https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCAmelCase_      							=     {
    """google/electra-small-generator""": 512,
    """google/electra-base-generator""": 512,
    """google/electra-large-generator""": 512,
    """google/electra-small-discriminator""": 512,
    """google/electra-base-discriminator""": 512,
    """google/electra-large-discriminator""": 512,
}

UpperCAmelCase_      							=     {
    """google/electra-small-generator""": {"""do_lower_case""": True},
    """google/electra-base-generator""": {"""do_lower_case""": True},
    """google/electra-large-generator""": {"""do_lower_case""": True},
    """google/electra-small-discriminator""": {"""do_lower_case""": True},
    """google/electra-base-discriminator""": {"""do_lower_case""": True},
    """google/electra-large-discriminator""": {"""do_lower_case""": True},
}
class 			UpperCamelCase_		(  _lowerCamelCase					):
					lowerCAmelCase_							=	VOCAB_FILES_NAMES
					lowerCAmelCase_							=	PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_							=	PRETRAINED_INIT_CONFIGURATION
					lowerCAmelCase_							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_							=	ElectraTokenizer




					def __init__(		self				,						lowerCAmelCase_=None				,						lowerCAmelCase_=None				,						lowerCAmelCase_=True				,						lowerCAmelCase_="[UNK]"				,						lowerCAmelCase_="[SEP]"				,						lowerCAmelCase_="[PAD]"				,						lowerCAmelCase_="[CLS]"				,						lowerCAmelCase_="[MASK]"				,						lowerCAmelCase_=True				,						lowerCAmelCase_=None				,						**lowerCAmelCase_				,						)		->							List[Any]:
										super().__init__(
										    lowerCAmelCase_				,						tokenizer_file=lowerCAmelCase_				,						do_lower_case=lowerCAmelCase_				,						unk_token=lowerCAmelCase_				,						sep_token=lowerCAmelCase_				,						pad_token=lowerCAmelCase_				,						cls_token=lowerCAmelCase_				,						mask_token=lowerCAmelCase_				,						tokenize_chinese_chars=lowerCAmelCase_				,						strip_accents=lowerCAmelCase_				,						**lowerCAmelCase_				,						)

										_snake_case									=				json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
										if (
										    normalizer_state.get('lowercase'				,						lowerCAmelCase_  ) != do_lower_case
										    or normalizer_state.get('strip_accents'				,						lowerCAmelCase_  ) != strip_accents
										    or normalizer_state.get('handle_chinese_chars'				,						lowerCAmelCase_  ) != tokenize_chinese_chars
										):
															_snake_case									=				getattr(lowerCAmelCase_				,						normalizer_state.pop('type'  )  )
															_snake_case									=				do_lower_case
															_snake_case									=				strip_accents
															_snake_case									=				tokenize_chinese_chars
															_snake_case									=				normalizer_class(**lowerCAmelCase_  )

										_snake_case									=				do_lower_case




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_=None  )		->							str:
										_snake_case									=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

										return output




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None  )		->							List[int]:
										_snake_case									=				[self.sep_token_id]
										_snake_case									=				[self.cls_token_id]
										if token_ids_a is None:
															return len(cls + token_ids_a + sep  ) * [0]
										return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None  )		->							Tuple[str]:
										_snake_case									=				self._tokenizer.model.save(lowerCAmelCase_				,						name=lowerCAmelCase_  )
										return tuple(lowerCAmelCase_  )










from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCAmelCase_      							=     {
    """configuration_luke""": ["""LUKE_PRETRAINED_CONFIG_ARCHIVE_MAP""", """LukeConfig"""],
    """tokenization_luke""": ["""LukeTokenizer"""],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCAmelCase_      							=     [
					    """LUKE_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """LukeForEntityClassification""",
					    """LukeForEntityPairClassification""",
					    """LukeForEntitySpanClassification""",
					    """LukeForMultipleChoice""",
					    """LukeForQuestionAnswering""",
					    """LukeForSequenceClassification""",
					    """LukeForTokenClassification""",
					    """LukeForMaskedLM""",
					    """LukeModel""",
					    """LukePreTrainedModel""",
					]


if TYPE_CHECKING:
					from .configuration_luke import LUKE_PRETRAINED_CONFIG_ARCHIVE_MAP, LukeConfig
					from .tokenization_luke import LukeTokenizer

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_luke import (
										    LUKE_PRETRAINED_MODEL_ARCHIVE_LIST,
										    LukeForEntityClassification,
										    LukeForEntityPairClassification,
										    LukeForEntitySpanClassification,
										    LukeForMaskedLM,
										    LukeForMultipleChoice,
										    LukeForQuestionAnswering,
										    LukeForSequenceClassification,
										    LukeForTokenClassification,
										    LukeModel,
										    LukePreTrainedModel,
										)

else:
					import sys

					UpperCAmelCase_      							=     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



