






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


SCREAMING_SNAKE_CASE 							=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE 							=						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE 							=						{
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

SCREAMING_SNAKE_CASE 							=						{
    'junnyu/roformer_chinese_small': 1536,
    'junnyu/roformer_chinese_base': 1536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


SCREAMING_SNAKE_CASE 							=						{
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}
class 							A_    (						__lowercase			):






 '''simple docstring'''




 _SCREAMING_SNAKE_CASE							:			Tuple =					VOCAB_FILES_NAMES
 _SCREAMING_SNAKE_CASE							:			Union[str, Any] =					PRETRAINED_VOCAB_FILES_MAP
 _SCREAMING_SNAKE_CASE							:			Union[str, Any] =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _SCREAMING_SNAKE_CASE							:			Any =					PRETRAINED_INIT_CONFIGURATION
 _SCREAMING_SNAKE_CASE							:			Any =					RoFormerTokenizer


 def __init__(	self							,  _A=None							,  _A=None							,  _A=True							,  _A="[UNK]"							,  _A="[SEP]"							,  _A="[PAD]"							,  _A="[CLS]"							,  _A="[MASK]"							,  _A=True							,  _A=None							,  **_A							,  ) ->		Tuple:

     """simple docstring"""



     super().__init__(
         _A							,  tokenizer_file=_A							,  do_lower_case=_A							,  unk_token=_A							,  sep_token=_A							,  pad_token=_A							,  cls_token=_A							,  mask_token=_A							,  tokenize_chinese_chars=_A							,  strip_accents=_A							,  **_A							,  )

     _UpperCAmelCase				:      Any			  =				json.loads(self.backend_tokenizer.normalizer.__getstate__())
     if (
         pre_tok_state.get('''lowercase'''							,  _A) != do_lower_case
         or pre_tok_state.get('''strip_accents'''							,  _A) != strip_accents
     ):
         _UpperCAmelCase				:      str			  =				getattr(_A							,  pre_tok_state.pop('''type'''))
         _UpperCAmelCase				:      Tuple			  =				do_lower_case
         _UpperCAmelCase				:      int			  =				strip_accents
         _UpperCAmelCase				:      List[str]			  =				pre_tok_class(**_A)

     _UpperCAmelCase				:      Tuple			  =				do_lower_case


 def __getstate__(	self) ->		Optional[int]:

     """simple docstring"""



     _UpperCAmelCase				:      List[str]			  =				self.__dict__.copy()
     _UpperCAmelCase				:      Union[str, Any]			  =				BertPreTokenizer()
     return state


 def __setstate__(	self							,  _A) ->		Any:

     """simple docstring"""



     _UpperCAmelCase				:      Tuple			  =				d
     _UpperCAmelCase				:      Tuple			  =				self.__dict__['''_tokenizer'''].get_vocab()
     _UpperCAmelCase				:      Tuple			  =				PreTokenizer.custom(JiebaPreTokenizer(_A))


 def  snake_case__				(	self							,  _A							,  _A=None) ->		Any:

     """simple docstring"""



     _UpperCAmelCase				:      Any			  =				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

     return output


 def  snake_case__				(	self							,  _A							,  _A = None) ->		List[int]:

     """simple docstring"""



     _UpperCAmelCase				:      Tuple			  =				[self.sep_token_id]
     _UpperCAmelCase				:      Optional[Any]			  =				[self.cls_token_id]
     if token_ids_a is None:
         return len(cls + token_ids_a + sep) * [0]
     return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


 def  snake_case__				(	self							,  _A							,  _A = None) ->		Tuple[str]:

     """simple docstring"""



     _UpperCAmelCase				:      Tuple			  =				self._tokenizer.model.save(_A							,  name=_A)
     return tuple(_A)


 def  snake_case__				(	self							,  _A							,  _A=None							,  _A=None							,  _A=False							,  **_A							,  ) ->		str:

     """simple docstring"""



     _UpperCAmelCase				:      Tuple			  =				BertPreTokenizer()
     return super().save_pretrained(_A							,  _A							,  _A							,  _A							,  **_A)












def      _lowerCamelCase	(						__A    :							int )      ->					bool:
    if not isinstance(__A		,		__A ):
        _UpperCAmelCase				:      Tuple			  =				f'''Input value of [number={number}] must be an integer'''
        raise TypeError(__A )
    if number < 0:
        return False
    _UpperCAmelCase				:      List[str]			  =				number * number
    while number > 0:
        if number % 10 != number_square % 10:
            return False
        number //= 10
        number_square //= 10
    return True


if __name__ == "__main__":
  import doctest

  doctest.testmod()




