


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__UpperCamelCase    		=       logging.get_logger(__name__)

__UpperCamelCase    		=       {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__UpperCamelCase    		=       {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

__UpperCamelCase    		=       {
    'junnyu/roformer_chinese_small': 1_5_3_6,
    'junnyu/roformer_chinese_base': 1_5_3_6,
    'junnyu/roformer_chinese_char_small': 5_1_2,
    'junnyu/roformer_chinese_char_base': 5_1_2,
    'junnyu/roformer_small_discriminator': 1_2_8,
    'junnyu/roformer_small_generator': 1_2_8,
}


__UpperCamelCase    		=       {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}







class        lowerCamelCase__	(							UpperCAmelCase      ):






			"""simple docstring"""






			_UpperCamelCase		:					Optional[int]    =		VOCAB_FILES_NAMES
			_UpperCamelCase		:					Optional[int]    =		PRETRAINED_VOCAB_FILES_MAP
			_UpperCamelCase		:					Union[str, Any]    =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCamelCase		:					Tuple    =		PRETRAINED_INIT_CONFIGURATION
			_UpperCamelCase		:					Tuple    =		RoFormerTokenizer




			def __init__(							self      ,						snake_case=None      ,						snake_case=None      ,						snake_case=True      ,						snake_case="[UNK]"      ,						snake_case="[SEP]"      ,						snake_case="[PAD]"      ,						snake_case="[CLS]"      ,						snake_case="[MASK]"      ,						snake_case=True      ,						snake_case=None      ,						**snake_case      ,						):

						'''simple docstring'''


						super().__init__(
						    snake_case      ,						tokenizer_file=snake_case      ,						do_lower_case=snake_case      ,						unk_token=snake_case      ,						sep_token=snake_case      ,						pad_token=snake_case      ,						cls_token=snake_case      ,						mask_token=snake_case      ,						tokenize_chinese_chars=snake_case      ,						strip_accents=snake_case      ,						**snake_case      ,						)

						UpperCamelCase__					=       json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
						if (
						    pre_tok_state.get("lowercase"      ,						snake_case     ) != do_lower_case
						    or pre_tok_state.get("strip_accents"      ,						snake_case     ) != strip_accents
						):
									UpperCamelCase__					=       getattr(snake_case      ,						pre_tok_state.pop("type"     )     )
									UpperCamelCase__					=       do_lower_case
									UpperCamelCase__					=       strip_accents
									UpperCamelCase__					=       pre_tok_class(**snake_case     )

						UpperCamelCase__					=       do_lower_case




			def __getstate__(							self     ):

						'''simple docstring'''


						UpperCamelCase__					=       self.__dict__.copy()
						UpperCamelCase__					=       BertPreTokenizer()
						return state




			def __setstate__(							self      ,						snake_case     ):

						'''simple docstring'''


						UpperCamelCase__					=       d
						UpperCamelCase__					=       self.__dict__["_tokenizer"].get_vocab()
						UpperCamelCase__					=       PreTokenizer.custom(JiebaPreTokenizer(snake_case     )     )




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case=None     ):

						'''simple docstring'''


						UpperCamelCase__					=       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

						return output




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case = None     ):

						'''simple docstring'''


						UpperCamelCase__					=       [self.sep_token_id]
						UpperCamelCase__					=       [self.cls_token_id]
						if token_ids_a is None:
									return len(cls + token_ids_a + sep     ) * [0]
						return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]




			def 				snake_case__				(							self      ,						snake_case      ,						snake_case = None     ):

						'''simple docstring'''


						UpperCamelCase__					=       self._tokenizer.model.save(snake_case      ,						name=snake_case     )
						return tuple(snake_case     )







			def 				snake_case__				(							self      ,						snake_case      ,						snake_case=None      ,						snake_case=None      ,						snake_case=False      ,						**snake_case      ,						):

						'''simple docstring'''


						UpperCamelCase__					=       BertPreTokenizer()
						return super().save_pretrained(snake_case      ,						snake_case      ,						snake_case      ,						snake_case      ,						**snake_case     )






def        UpperCamelCase_(				_A   :int				,     _A   :int  )->		str:
			if number < 0 or shift_amount < 0:
						raise ValueError("both inputs must be positive integers"  )

			UpperCamelCase__					=       str(bin(_A  )  )
			binary_number += "0" * shift_amount
			return binary_number
def        UpperCamelCase_(				_A   :int				,     _A   :int  )->		str:
			if number < 0 or shift_amount < 0:
						raise ValueError("both inputs must be positive integers"  )

			UpperCamelCase__					=       str(bin(_A  )  )[2:]
			if shift_amount >= len(_A  ):
						return "0b0"
			UpperCamelCase__					=       binary_number[: len(_A  ) - shift_amount]
			return "0b" + shifted_binary_number
def        UpperCamelCase_(				_A   :int				,     _A   :int  )->		str:
			if number >= 0:  # Get binary representation of positive number
						UpperCamelCase__					=       "0" + str(bin(_A  )  ).strip("-"  )[2:]
			else:  # Get binary (2's complement) representation of negative number
						UpperCamelCase__					=       len(bin(_A  )[3:]  )  # Find 2's complement of number
						UpperCamelCase__					=       bin(abs(_A  ) - (1 << binary_number_length)  )[3:]
						UpperCamelCase__					=       (
						    "1" + "0" * (binary_number_length - len(_A  )) + binary_number
						)

			if shift_amount >= len(_A  ):
						return "0b" + binary_number[0] * len(_A  )
			return (
			    "0b"
			    + binary_number[0] * shift_amount
			    + binary_number[: len(_A  ) - shift_amount]
			)


if __name__ == "__main__":
						import doctest

						doctest.testmod()


