





"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


a_			=      {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

a_			=      {
    """vocab_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-generator""": """https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt""",
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-generator""": (
            """https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"""
        ),
    },
}

a_			=      {
    """google/electra-small-generator""": 512,
    """google/electra-base-generator""": 512,
    """google/electra-large-generator""": 512,
    """google/electra-small-discriminator""": 512,
    """google/electra-base-discriminator""": 512,
    """google/electra-large-discriminator""": 512,
}

a_			=      {
    """google/electra-small-generator""": {"""do_lower_case""": True},
    """google/electra-base-generator""": {"""do_lower_case""": True},
    """google/electra-large-generator""": {"""do_lower_case""": True},
    """google/electra-small-discriminator""": {"""do_lower_case""": True},
    """google/electra-base-discriminator""": {"""do_lower_case""": True},
    """google/electra-large-discriminator""": {"""do_lower_case""": True},
}






class 		__snake_case (		SCREAMING_SNAKE_CASE__					):





 """simple docstring"""


 _lowerCamelCase				=					VOCAB_FILES_NAMES
 _lowerCamelCase				=					PRETRAINED_VOCAB_FILES_MAP
 _lowerCamelCase				=					PRETRAINED_INIT_CONFIGURATION
 _lowerCamelCase				=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCamelCase				=					ElectraTokenizer

 def __init__(   self ,						__lowerCamelCase=None ,						__lowerCamelCase=None ,						__lowerCamelCase=True ,						__lowerCamelCase="[UNK]" ,						__lowerCamelCase="[SEP]" ,						__lowerCamelCase="[PAD]" ,						__lowerCamelCase="[CLS]" ,						__lowerCamelCase="[MASK]" ,						__lowerCamelCase=True ,						__lowerCamelCase=None ,						**__lowerCamelCase ,						):



     '''simple docstring'''




     super().__init__(
         __lowerCamelCase ,						tokenizer_file=__lowerCamelCase ,						do_lower_case=__lowerCamelCase ,						unk_token=__lowerCamelCase ,						sep_token=__lowerCamelCase ,						pad_token=__lowerCamelCase ,						cls_token=__lowerCamelCase ,						mask_token=__lowerCamelCase ,						tokenize_chinese_chars=__lowerCamelCase ,						strip_accents=__lowerCamelCase ,						**__lowerCamelCase ,						)

     __A   :						List[str]									=					json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
     if (
         normalizer_state.get('''lowercase''' ,						__lowerCamelCase  ) != do_lower_case
         or normalizer_state.get('''strip_accents''' ,						__lowerCamelCase  ) != strip_accents
         or normalizer_state.get('''handle_chinese_chars''' ,						__lowerCamelCase  ) != tokenize_chinese_chars
     ):
         __A   :						Any									=					getattr(__lowerCamelCase ,						normalizer_state.pop('''type'''  )  )
         __A   :						Dict									=					do_lower_case
         __A   :						Dict									=					strip_accents
         __A   :						Dict									=					tokenize_chinese_chars
         __A   :						Union[str, Any]									=					normalizer_class(**__lowerCamelCase  )

     __A   :						List[str]									=					do_lower_case

 def  UpperCamelCase__(   self ,						__lowerCamelCase ,						__lowerCamelCase=None  ):



     '''simple docstring'''




     __A   :						str									=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

     return output

 def  UpperCamelCase__(   self ,						__lowerCamelCase ,						__lowerCamelCase = None  ):



     '''simple docstring'''




     __A   :						List[Any]									=					[self.sep_token_id]
     __A   :						List[Any]									=					[self.cls_token_id]
     if token_ids_a is None:
         return len(cls + token_ids_a + sep  ) * [0]
     return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







 def  UpperCamelCase__(   self ,						__lowerCamelCase ,						__lowerCamelCase = None  ):



     '''simple docstring'''




     __A   :						Dict									=					self._tokenizer.model.save(__lowerCamelCase ,						name=__lowerCamelCase  )
     return tuple(__lowerCamelCase  )











"""simple docstring"""



from typing import TYPE_CHECKING

from ...file_utils import _LazyModule, is_tokenizers_available, is_torch_available
from ...utils import OptionalDependencyNotAvailable


a_			=      {"""configuration_gpt_neox""": ["""GPT_NEOX_PRETRAINED_CONFIG_ARCHIVE_MAP""", """GPTNeoXConfig"""]}

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a_			=      ["""GPTNeoXTokenizerFast"""]

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a_			=      [
           """GPT_NEOX_PRETRAINED_MODEL_ARCHIVE_LIST""",
           """GPTNeoXForCausalLM""",
           """GPTNeoXForQuestionAnswering""",
           """GPTNeoXForSequenceClassification""",
           """GPTNeoXForTokenClassification""",
           """GPTNeoXLayer""",
           """GPTNeoXModel""",
           """GPTNeoXPreTrainedModel""",
       ]


if TYPE_CHECKING:
       from .configuration_gpt_neox import GPT_NEOX_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTNeoXConfig

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_gpt_neox_fast import GPTNeoXTokenizerFast

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_gpt_neox import (
                  GPT_NEOX_PRETRAINED_MODEL_ARCHIVE_LIST,
                  GPTNeoXForCausalLM,
                  GPTNeoXForQuestionAnswering,
                  GPTNeoXForSequenceClassification,
                  GPTNeoXForTokenClassification,
                  GPTNeoXLayer,
                  GPTNeoXModel,
                  GPTNeoXPreTrainedModel,
              )


else:
       import sys

       a_			=      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




