





# Lint as: python3
import itertools
import os
import re


A__:    List[Any]  	=	re.compile(R'''([A-Z]+)([A-Z][a-z])''')
A__:    Union[str, Any]  	=	re.compile(R'''([a-z\d])([A-Z])''')

A__:    Union[str, Any]  	=	re.compile(R'''(?<!_)_(?!_)''')
A__:    List[Any]  	=	re.compile(R'''(_{2,})''')

A__:    List[str]  	=	R'''^\w+(\.\w+)*$'''

A__:    Optional[int]  	=	R'''<>:/\|?*'''



def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Union[str, Any]					=     _uppercase_uppercase_re.sub(R"\1_\2"    ,A_)
    UpperCamelCase__:	Union[str, Any]					=     _lowercase_uppercase_re.sub(R"\1_\2"    ,A_)
    return name.lower()



def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Union[str, Any]					=     _single_underscore_re.split(A_)
    UpperCamelCase__:	Any					=     [_multiple_underscores_re.split(A_) for n in name]
    return "".join(n.capitalize() for n in itertools.chain.from_iterable(A_) if n != "")



def 				lowerCAmelCase_ (							A_):
    if os.path.basename(A_) != name:
        raise ValueError(F"Should be a dataset name, not a path: {name}")
    return camelcase_to_snakecase(A_)



def 				lowerCAmelCase_ (							A_    ,A_):
    if os.path.basename(A_) != name:
        raise ValueError(F"Should be a dataset name, not a path: {name}")
    if not re.match(_split_re    ,A_):
        raise ValueError(F"Split name should match '{_split_re}'' but got '{split}'.")
    return F"{filename_prefix_for_name(A_)}-{split}"



def 				lowerCAmelCase_ (							A_    ,A_    ,A_    ,A_=None):
    UpperCamelCase__:	str					=     filename_prefix_for_split(A_    ,A_)
    if filetype_suffix:
        prefix += F".{filetype_suffix}"
    UpperCamelCase__:	Optional[Any]					=     os.path.join(A_    ,A_)
    return F"{filepath}*"




def 				lowerCAmelCase_ (							A_    ,A_    ,A_    ,A_=None    ,A_=None):
    UpperCamelCase__:	Union[str, Any]					=     filename_prefix_for_split(A_    ,A_)
    UpperCamelCase__:	Optional[int]					=     os.path.join(A_    ,A_)

    if shard_lengths:
        UpperCamelCase__:	Dict					=     len(A_)
        UpperCamelCase__:	Optional[int]					=     [F"{prefix}-{shard_id:05d}-of-{num_shards:05d}" for shard_id in range(A_)]
        if filetype_suffix:
            UpperCamelCase__:	List[Any]					=     [filename + F".{filetype_suffix}" for filename in filenames]
        return filenames
    else:
        UpperCamelCase__:	List[str]					=     prefix
        if filetype_suffix:
            filename += F".{filetype_suffix}"
        return [filename]








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


A__:    Dict  	=	{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A__:    Any  	=	{
    '''vocab_file''': {
        '''google/electra-small-generator''': (
            '''https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt'''
        ),
        '''google/electra-base-generator''': '''https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt''',
        '''google/electra-large-generator''': (
            '''https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt'''
        ),
        '''google/electra-small-discriminator''': (
            '''https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt'''
        ),
        '''google/electra-base-discriminator''': (
            '''https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt'''
        ),
        '''google/electra-large-discriminator''': (
            '''https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''google/electra-small-generator''': (
            '''https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-base-generator''': (
            '''https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-large-generator''': (
            '''https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-small-discriminator''': (
            '''https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-base-discriminator''': (
            '''https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-large-discriminator''': (
            '''https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json'''
        ),
    },
}

A__:    Tuple  	=	{
    '''google/electra-small-generator''': 512,
    '''google/electra-base-generator''': 512,
    '''google/electra-large-generator''': 512,
    '''google/electra-small-discriminator''': 512,
    '''google/electra-base-discriminator''': 512,
    '''google/electra-large-discriminator''': 512,
}

A__:    List[Any]  	=	{
    '''google/electra-small-generator''': {'''do_lower_case''': True},
    '''google/electra-base-generator''': {'''do_lower_case''': True},
    '''google/electra-large-generator''': {'''do_lower_case''': True},
    '''google/electra-small-discriminator''': {'''do_lower_case''': True},
    '''google/electra-base-discriminator''': {'''do_lower_case''': True},
    '''google/electra-large-discriminator''': {'''do_lower_case''': True},
}




class        _a    (    UpperCamelCase__):

      """simple docstring"""


      UpperCamelCase__  		=       VOCAB_FILES_NAMES
      UpperCamelCase__  		=       PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase__  		=       PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase__  		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase__  		=       ElectraTokenizer






      def __init__(		self:			List[str]	,    __lowerCamelCase:			int=None	,    __lowerCamelCase:			List[str]=None	,    __lowerCamelCase:			str=True	,    __lowerCamelCase:			Dict="[UNK]"	,    __lowerCamelCase:			Tuple="[SEP]"	,    __lowerCamelCase:			Optional[int]="[PAD]"	,    __lowerCamelCase:			str="[CLS]"	,    __lowerCamelCase:			List[str]="[MASK]"	,    __lowerCamelCase:			Optional[Any]=True	,    __lowerCamelCase:			Optional[Any]=None	,    **__lowerCamelCase:			str	,    ):
          '''simple docstring'''


          super().__init__(
              __lowerCamelCase	,    tokenizer_file=__lowerCamelCase	,    do_lower_case=__lowerCamelCase	,    unk_token=__lowerCamelCase	,    sep_token=__lowerCamelCase	,    pad_token=__lowerCamelCase	,    cls_token=__lowerCamelCase	,    mask_token=__lowerCamelCase	,    tokenize_chinese_chars=__lowerCamelCase	,    strip_accents=__lowerCamelCase	,    **__lowerCamelCase	,    )

          UpperCamelCase__:	Any					=     json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
          if (
              normalizer_state.get("lowercase"	,    __lowerCamelCase					) != do_lower_case
              or normalizer_state.get("strip_accents"	,    __lowerCamelCase					) != strip_accents
              or normalizer_state.get("handle_chinese_chars"	,    __lowerCamelCase					) != tokenize_chinese_chars
          ):
              UpperCamelCase__:	Any					=     getattr(__lowerCamelCase	,    normalizer_state.pop("type"					)					)
              UpperCamelCase__:	Union[str, Any]					=     do_lower_case
              UpperCamelCase__:	str					=     strip_accents
              UpperCamelCase__:	Any					=     tokenize_chinese_chars
              UpperCamelCase__:	Optional[Any]					=     normalizer_class(**__lowerCamelCase					)

          UpperCamelCase__:	Union[str, Any]					=     do_lower_case






      def 			UpperCAmelCase_       (		self:			Optional[int]	,    __lowerCamelCase:			List[str]	,    __lowerCamelCase:			Tuple=None					):
          '''simple docstring'''


          UpperCamelCase__:	Optional[Any]					=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

          return output






      def 			UpperCAmelCase_       (		self:			Tuple	,    __lowerCamelCase:			List[int]	,    __lowerCamelCase:			Optional[List[int]] = None					):
          '''simple docstring'''


          UpperCamelCase__:	Tuple					=     [self.sep_token_id]
          UpperCamelCase__:	Union[str, Any]					=     [self.cls_token_id]
          if token_ids_a is None:
              return len(cls + token_ids_a + sep					) * [0]
          return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]






      def 			UpperCAmelCase_       (		self:			Any	,    __lowerCamelCase:			str	,    __lowerCamelCase:			Optional[str] = None					):
          '''simple docstring'''


          UpperCamelCase__:	List[Any]					=     self._tokenizer.model.save(__lowerCamelCase	,    name=__lowerCamelCase					)
          return tuple(__lowerCamelCase					)

