


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


__UpperCamelCase       :       Dict						       =				{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__UpperCamelCase       :       Union[str, Any]						       =				{
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

__UpperCamelCase       :       Optional[int]						       =				{
    "google/electra-small-generator": 512,
    "google/electra-base-generator": 512,
    "google/electra-large-generator": 512,
    "google/electra-small-discriminator": 512,
    "google/electra-base-discriminator": 512,
    "google/electra-large-discriminator": 512,
}

__UpperCamelCase       :       List[Any]						       =				{
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}

class       __magic_name__	(     __lowerCAmelCase):
     A:		Optional[int]			=      VOCAB_FILES_NAMES
     A:		List[Any]			=      PRETRAINED_VOCAB_FILES_MAP
     A:		Optional[int]			=      PRETRAINED_INIT_CONFIGURATION
     A:		List[str]			=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     A:		List[str]			=      ElectraTokenizer


     def __init__(      self							:     Optional[Any]							,	lowerCamelCase__							:     Tuple=None							,	lowerCamelCase__							:     Tuple=None							,	lowerCamelCase__							:     Union[str, Any]=True							,	lowerCamelCase__							:     Tuple="[UNK]"							,	lowerCamelCase__							:     str="[SEP]"							,	lowerCamelCase__							:     Union[str, Any]="[PAD]"							,	lowerCamelCase__							:     Any="[CLS]"							,	lowerCamelCase__							:     List[Any]="[MASK]"							,	lowerCamelCase__							:     Any=True							,	lowerCamelCase__							:     Any=None							,	**lowerCamelCase__							:     Dict							,	)  ->      Any:





      '''simple docstring'''






      super().__init__(
          lowerCamelCase__							,	tokenizer_file=lowerCamelCase__							,	do_lower_case=lowerCamelCase__							,	unk_token=lowerCamelCase__							,	sep_token=lowerCamelCase__							,	pad_token=lowerCamelCase__							,	cls_token=lowerCamelCase__							,	mask_token=lowerCamelCase__							,	tokenize_chinese_chars=lowerCamelCase__							,	strip_accents=lowerCamelCase__							,	**lowerCamelCase__							,	)

      UpperCamelCase__					:    Union[str, Any]          =   json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
      if (
          normalizer_state.get('''lowercase'''							,	lowerCamelCase__			) != do_lower_case
          or normalizer_state.get('''strip_accents'''							,	lowerCamelCase__			) != strip_accents
          or normalizer_state.get('''handle_chinese_chars'''							,	lowerCamelCase__			) != tokenize_chinese_chars
      ):
       UpperCamelCase__					:    Dict          =   getattr(lowerCamelCase__							,	normalizer_state.pop('''type'''			)			)
       UpperCamelCase__					:    Optional[Any]          =   do_lower_case
       UpperCamelCase__					:    int          =   strip_accents
       UpperCamelCase__					:    Any          =   tokenize_chinese_chars
       UpperCamelCase__					:    Optional[int]          =   normalizer_class(**lowerCamelCase__			)

      UpperCamelCase__					:    Dict          =   do_lower_case


     def      UpperCAmelCase__	(      self							:     int							,	lowerCamelCase__							:     List[Any]							,	lowerCamelCase__							:     Dict=None			)  ->      Union[str, Any]:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output


     def      UpperCAmelCase__	(      self							:     int							,	lowerCamelCase__							:     List[int]							,	lowerCamelCase__							:     Optional[List[int]] = None			)  ->      List[int]:





      '''simple docstring'''






      UpperCamelCase__					:    Union[str, Any]          =   [self.sep_token_id]
      UpperCamelCase__					:    Optional[int]          =   [self.cls_token_id]
      if token_ids_a is None:
       return len(cls + token_ids_a + sep			) * [0]
      return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]


     def      UpperCAmelCase__	(      self							:     Optional[Any]							,	lowerCamelCase__							:     str							,	lowerCamelCase__							:     Optional[str] = None			)  ->      Tuple[str]:





      '''simple docstring'''






      UpperCamelCase__					:    Any          =   self._tokenizer.model.save(lowerCamelCase__							,	name=lowerCamelCase__			)
      return tuple(lowerCamelCase__			)






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


__UpperCamelCase       :       Optional[int]						       =				{
    "configuration_rag": ["RagConfig"],
    "retrieval_rag": ["RagRetriever"],
    "tokenization_rag": ["RagTokenizer"],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __UpperCamelCase       :       Optional[int]						       =				[
           "RagModel",
           "RagPreTrainedModel",
           "RagSequenceForGeneration",
           "RagTokenForGeneration",
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __UpperCamelCase       :       List[Any]						       =				[
           "TFRagModel",
           "TFRagPreTrainedModel",
           "TFRagSequenceForGeneration",
           "TFRagTokenForGeneration",
       ]


if TYPE_CHECKING:
       from .configuration_rag import RagConfig
       from .retrieval_rag import RagRetriever
       from .tokenization_rag import RagTokenizer

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_rag import RagModel, RagPreTrainedModel, RagSequenceForGeneration, RagTokenForGeneration

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_rag import (
                  TFRagModel,
                  TFRagPreTrainedModel,
                  TFRagSequenceForGeneration,
                  TFRagTokenForGeneration,
              )

else:
       import sys

       __UpperCamelCase       :       List[Any]						       =				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


