




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


a_							=    logging.get_logger(__name__)

a_							=    {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

a_							=    {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

a_							=    {
    'junnyu/roformer_chinese_small': 1536,
    'junnyu/roformer_chinese_base': 1536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


a_							=    {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}


class      lowercase__     (			UpperCamelCase_	):
	a_         =VOCAB_FILES_NAMES
	a_         =PRETRAINED_VOCAB_FILES_MAP
	a_         =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a_         =PRETRAINED_INIT_CONFIGURATION
	a_         =RoFormerTokenizer

	def __init__(							self		,    __UpperCAmelCase=None		,    __UpperCAmelCase=None		,    __UpperCAmelCase=True		,    __UpperCAmelCase="[UNK]"		,    __UpperCAmelCase="[SEP]"		,    __UpperCAmelCase="[PAD]"		,    __UpperCAmelCase="[CLS]"		,    __UpperCAmelCase="[MASK]"		,    __UpperCAmelCase=True		,    __UpperCAmelCase=None		,    **__UpperCAmelCase		,    )->					Union[str, Any]:







							'''simple docstring'''





							super().__init__(
							    __UpperCAmelCase		,    tokenizer_file=__UpperCAmelCase		,    do_lower_case=__UpperCAmelCase		,    unk_token=__UpperCAmelCase		,    sep_token=__UpperCAmelCase		,    pad_token=__UpperCAmelCase		,    cls_token=__UpperCAmelCase		,    mask_token=__UpperCAmelCase		,    tokenize_chinese_chars=__UpperCAmelCase		,    strip_accents=__UpperCAmelCase		,    **__UpperCAmelCase		,    )

							lowerCAmelCase__       =						json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
							if (
							    pre_tok_state.get("lowercase"		,    __UpperCAmelCase       ) != do_lower_case
							    or pre_tok_state.get("strip_accents"		,    __UpperCAmelCase       ) != strip_accents
							):
													lowerCAmelCase__       =						getattr(__UpperCAmelCase		,    pre_tok_state.pop("type"       )       )
													lowerCAmelCase__       =						do_lower_case
													lowerCAmelCase__       =						strip_accents
													lowerCAmelCase__       =						pre_tok_class(**__UpperCAmelCase       )

							lowerCAmelCase__       =						do_lower_case

	def __getstate__(							self       )->					str:







							'''simple docstring'''





							lowerCAmelCase__       =						self.__dict__.copy()
							lowerCAmelCase__       =						BertPreTokenizer()
							return state

	def __setstate__(							self		,    __UpperCAmelCase       )->					Tuple:







							'''simple docstring'''





							lowerCAmelCase__       =						d
							lowerCAmelCase__       =						self.__dict__['''_tokenizer'''].get_vocab()
							lowerCAmelCase__       =						PreTokenizer.custom(JiebaPreTokenizer(__UpperCAmelCase       )       )

	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase		,    __UpperCAmelCase=None       )->					Optional[Any]:







							'''simple docstring'''





							lowerCAmelCase__       =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

							return output

	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase		,    __UpperCAmelCase = None       )->					List[int]:







							'''simple docstring'''





							lowerCAmelCase__       =						[self.sep_token_id]
							lowerCAmelCase__       =						[self.cls_token_id]
							if token_ids_a is None:
													return len(cls + token_ids_a + sep       ) * [0]
							return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]

	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase		,    __UpperCAmelCase = None       )->					Tuple[str]:







							'''simple docstring'''





							lowerCAmelCase__       =						self._tokenizer.model.save(__UpperCAmelCase		,    name=__UpperCAmelCase       )
							return tuple(__UpperCAmelCase       )


	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase		,    __UpperCAmelCase=None		,    __UpperCAmelCase=None		,    __UpperCAmelCase=False		,    **__UpperCAmelCase		,    )->					Dict:







							'''simple docstring'''





							lowerCAmelCase__       =						BertPreTokenizer()
							return super().save_pretrained(__UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    **__UpperCAmelCase       )














def        _a  (     UpperCamelCase_					:				list    ,   UpperCamelCase_					:				list   ) ->   float:
						"""simple docstring"""







						_validate_point(UpperCamelCase_   )
						_validate_point(UpperCamelCase_   )
						if len(UpperCamelCase_   ) != len(UpperCamelCase_   ):
												raise ValueError("Both points must be in the same n-dimensional space"   )

						return float(sum(abs(a - b   ) for a, b in zip(UpperCamelCase_    ,   UpperCamelCase_   )   )   )


def        _a  (     UpperCamelCase_					:				list[float]   ) ->   None:
						"""simple docstring"""







						if point:
												if isinstance(UpperCamelCase_    ,   UpperCamelCase_   ):
																		for item in point:
																								if not isinstance(UpperCamelCase_    ,   (int, float)   ):
																														lowerCAmelCase__       =						(
																														    "Expected a list of numbers as input, found "
																														    F"{type(UpperCamelCase_   ).__name__}"
																														)
																														raise TypeError(UpperCamelCase_   )
												else:
																		lowerCAmelCase__       =						F"Expected a list of numbers as input, found {type(UpperCamelCase_   ).__name__}"
																		raise TypeError(UpperCamelCase_   )
						else:
												raise ValueError("Missing an input"   )


def        _a  (     UpperCamelCase_					:				list    ,   UpperCamelCase_					:				list   ) ->   float:
						"""simple docstring"""







						_validate_point(UpperCamelCase_   )
						_validate_point(UpperCamelCase_   )
						if len(UpperCamelCase_   ) != len(UpperCamelCase_   ):
												raise ValueError("Both points must be in the same n-dimensional space"   )

						return float(sum(abs(x - y   ) for x, y in zip(UpperCamelCase_    ,   UpperCamelCase_   )   )   )


if __name__ == "__main__":
	import doctest

	doctest.testmod()






