





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


lowerCamelCase__          =    logging.get_logger(__name__)

lowerCamelCase__          =    {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase__          =    {
    '''vocab_file''': {
        '''junnyu/roformer_chinese_small''': '''https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_base''': '''https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_char_small''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_chinese_char_base''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_discriminator''': (
            '''https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_generator''': (
            '''https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'''
        ),
    }
}

lowerCamelCase__          =    {
    '''junnyu/roformer_chinese_small''': 15_36,
    '''junnyu/roformer_chinese_base''': 15_36,
    '''junnyu/roformer_chinese_char_small''': 5_12,
    '''junnyu/roformer_chinese_char_base''': 5_12,
    '''junnyu/roformer_small_discriminator''': 1_28,
    '''junnyu/roformer_small_generator''': 1_28,
}


lowerCamelCase__          =    {
    '''junnyu/roformer_chinese_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_discriminator''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_generator''': {'''do_lower_case''': True},
}
class 	__magic_name__			(lowerCamelCase_ ):
					lowerCamelCase__     						=     VOCAB_FILES_NAMES
					lowerCamelCase__     						=     PRETRAINED_VOCAB_FILES_MAP
					lowerCamelCase__     						=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCamelCase__     						=     PRETRAINED_INIT_CONFIGURATION
					lowerCamelCase__     						=     RoFormerTokenizer




					def __init__(       self	,   _a=None	,   _a=None	,   _a=True	,   _a="[UNK]"	,   _a="[SEP]"	,   _a="[PAD]"	,   _a="[CLS]"	,   _a="[MASK]"	,   _a=True	,   _a=None	,   **_a	,   )			->					Tuple:
							super().__init__(
							    snake_case__	,   tokenizer_file=snake_case__	,   do_lower_case=snake_case__	,   unk_token=snake_case__	,   sep_token=snake_case__	,   pad_token=snake_case__	,   cls_token=snake_case__	,   mask_token=snake_case__	,   tokenize_chinese_chars=snake_case__	,   strip_accents=snake_case__	,   **snake_case__	,   )

							lowerCAmelCase_												=				json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
							if (
							    pre_tok_state.get("lowercase"	,   snake_case__						) != do_lower_case
							    or pre_tok_state.get("strip_accents"	,   snake_case__						) != strip_accents
							):
									lowerCAmelCase_												=				getattr(snake_case__	,   pre_tok_state.pop("type"						)						)
									lowerCAmelCase_												=				do_lower_case
									lowerCAmelCase_												=				strip_accents
									lowerCAmelCase_												=				pre_tok_class(**snake_case__						)

							lowerCAmelCase_												=				do_lower_case




					def __getstate__(       self						)			->					List[str]:
							lowerCAmelCase_												=				self.__dict__.copy()
							lowerCAmelCase_												=				BertPreTokenizer()
							return state




					def __setstate__(       self	,   _a						)			->					Dict:
							lowerCAmelCase_												=				d
							lowerCAmelCase_												=				self.__dict__['_tokenizer'].get_vocab()
							lowerCAmelCase_												=				PreTokenizer.custom(JiebaPreTokenizer(snake_case__						)						)




					def 					__a     (       self	,   _a	,   _a=None						)			->					Any:
							lowerCAmelCase_												=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

							return output




					def 					__a     (       self	,   _a	,   _a = None						)			->					Optional[int]:
							lowerCAmelCase_												=				[self.sep_token_id]
							lowerCAmelCase_												=				[self.cls_token_id]
							if token_ids_a is None:
									return len(cls + token_ids_a + sep						) * [0]
							return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]




					def 					__a     (       self	,   _a	,   _a = None						)			->					int:
							lowerCAmelCase_												=				self._tokenizer.model.save(snake_case__	,   name=snake_case__						)
							return tuple(snake_case__						)




					def 					__a     (       self	,   _a	,   _a=None	,   _a=None	,   _a=False	,   **_a	,   )			->					Optional[int]:
							lowerCAmelCase_												=				BertPreTokenizer()
							return super().save_pretrained(snake_case__	,   snake_case__	,   snake_case__	,   snake_case__	,   **snake_case__						)







def        __UpperCAmelCase						(							lowerCamelCase_ :     str  )       ->  bool:







			"""simple docstring"""





			SCREAMING_SNAKE_CASE_   : Any					  =      0
			for ch in input_str:
						SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      ord(lowerCamelCase_  )
						SCREAMING_SNAKE_CASE_   : Tuple					  =      pow(2  ,							lowerCamelCase_  )

						# If we already turned on bit for current character's unicode
						if bitmap >> ch_unicode & 1 == 1:
									return False
						bitmap |= ch_bit_index_on
			return True


if __name__ == "__main__":
						import doctest

						doctest.testmod()



