



from ..utils import DummyObject, requires_backends







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Union[str, Any]			,		*A       :  Any			,		**A       :  int)       ->   List[str]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  int			,		*A       :  Optional[int]			,		**A       :  Dict)       ->   Union[str, Any]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  int			,		*A       :  Optional[int]			,		**A       :  List[Any])       ->   str:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  str			,		*A       :  str			,		**A       :  Dict)       ->   List[str]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  List[Any]			,		*A       :  Any			,		**A       :  Union[str, Any])       ->   Union[str, Any]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  int			,		*A       :  int			,		**A       :  Any)       ->   Union[str, Any]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Optional[Any]			,		*A       :  Tuple			,		**A       :  List[Any])       ->   str:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Union[str, Any]			,		*A       :  Dict			,		**A       :  Any)       ->   List[Any]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Any			,		*A       :  Dict			,		**A       :  int)       ->   Dict:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  int			,		*A       :  str			,		**A       :  str)       ->   Optional[Any]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Union[str, Any]			,		*A       :  List[str]			,		**A       :  int)       ->   Any:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Optional[int]			,		*A       :  Optional[Any]			,		**A       :  Tuple)       ->   int:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Optional[Any]			,		*A       :  int			,		**A       :  int)       ->   Optional[Any]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Optional[int]			,		*A       :  str			,		**A       :  str)       ->   List[Any]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Union[str, Any]			,		*A       :  Dict			,		**A       :  Optional[int])       ->   Any:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Union[str, Any]			,		*A       :  Union[str, Any]			,		**A       :  List[Any])       ->   Optional[int]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  List[Any]			,		*A       :  Optional[Any]			,		**A       :  Any)       ->   Union[str, Any]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Optional[Any]			,		*A       :  str			,		**A       :  int)       ->   Any:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Optional[Any]			,		*A       :  Dict			,		**A       :  str)       ->   List[str]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  str			,		*A       :  Any			,		**A       :  str)       ->   List[Any]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  List[Any]			,		*A       :  Optional[int]			,		**A       :  Tuple)       ->   Optional[int]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  str			,		*A       :  Optional[Any]			,		**A       :  List[Any])       ->   str:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Tuple			,		*A       :  str			,		**A       :  int)       ->   str:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  int			,		*A       :  Optional[int]			,		**A       :  List[str])       ->   Union[str, Any]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Optional[Any]			,		*A       :  Union[str, Any]			,		**A       :  List[str])       ->   Tuple:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  str			,		*A       :  Optional[int]			,		**A       :  Dict)       ->   List[Any]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Dict			,		*A       :  Tuple			,		**A       :  Dict)       ->   Optional[int]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  str			,		*A       :  str			,		**A       :  Optional[Any])       ->   Tuple:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Optional[Any]			,		*A       :  int			,		**A       :  List[str])       ->   int:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  Any			,		*A       :  int			,		**A       :  int)       ->   int:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])







class       __lowerCAmelCase   (      metaclass=A						):
			UpperCamelCase        =					['''sentencepiece''']



			def __init__(						self       :  str			,		*A       :  Optional[int]			,		**A       :  Optional[Any])       ->   List[str]:



						"""simple docstring"""






						requires_backends(self			,		['sentencepiece'])





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCAmelCase__       =	logging.get_logger(__name__)

UpperCAmelCase__       =	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__       =	{
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

UpperCAmelCase__       =	{
    "junnyu/roformer_chinese_small": 1536,
    "junnyu/roformer_chinese_base": 1536,
    "junnyu/roformer_chinese_char_small": 512,
    "junnyu/roformer_chinese_char_base": 512,
    "junnyu/roformer_small_discriminator": 128,
    "junnyu/roformer_small_generator": 128,
}


UpperCAmelCase__       =	{
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}







class       __lowerCAmelCase   (      A						):
			UpperCamelCase        =					VOCAB_FILES_NAMES
			UpperCamelCase        =					PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase        =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase        =					PRETRAINED_INIT_CONFIGURATION
			UpperCamelCase        =					RoFormerTokenizer



			def __init__(						self       :  Dict			,		A       :  Any=None			,		A       :  Optional[Any]=None			,		A       :  Union[str, Any]=True			,		A       :  List[str]="[UNK]"			,		A       :  List[str]="[SEP]"			,		A       :  Union[str, Any]="[PAD]"			,		A       :  Any="[CLS]"			,		A       :  str="[MASK]"			,		A       :  Optional[int]=True			,		A       :  str=None			,		**A       :  Dict			,		)       ->   Union[str, Any]:



						"""simple docstring"""






						super().__init__(
						    A			,		tokenizer_file=A			,		do_lower_case=A			,		unk_token=A			,		sep_token=A			,		pad_token=A			,		cls_token=A			,		mask_token=A			,		tokenize_chinese_chars=A			,		strip_accents=A			,		**A			,		)

						_UpperCAmelCase			=       json.loads(self.backend_tokenizer.normalizer.__getstate__())
						if (
						    pre_tok_state.get('lowercase'			,		A) != do_lower_case
						    or pre_tok_state.get('strip_accents'			,		A) != strip_accents
						):
									_UpperCAmelCase			=       getattr(A			,		pre_tok_state.pop('type'))
									_UpperCAmelCase			=       do_lower_case
									_UpperCAmelCase			=       strip_accents
									_UpperCAmelCase			=       pre_tok_class(**A)

						_UpperCAmelCase			=       do_lower_case



			def __getstate__(						self       :  Optional[Any])       ->   Optional[Any]:



						"""simple docstring"""






						_UpperCAmelCase			=       self.__dict__.copy()
						_UpperCAmelCase			=       BertPreTokenizer()
						return state



			def __setstate__(						self       :  List[Any]			,		A       :  Optional[Any])       ->   List[str]:



						"""simple docstring"""






						_UpperCAmelCase			=       d
						_UpperCAmelCase			=       self.__dict__['_tokenizer'].get_vocab()
						_UpperCAmelCase			=       PreTokenizer.custom(JiebaPreTokenizer(A))



			def    _lowerCamelCase	(						self       :  List[Any]			,		A       :  Optional[int]			,		A       :  List[Any]=None)       ->   Optional[Any]:



						"""simple docstring"""






						_UpperCAmelCase			=       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

						return output



			def    _lowerCamelCase	(						self       :  List[str]			,		A       :  List[int]			,		A       :  Optional[List[int]] = None)       ->   List[int]:



						"""simple docstring"""






						_UpperCAmelCase			=       [self.sep_token_id]
						_UpperCAmelCase			=       [self.cls_token_id]
						if token_ids_a is None:
									return len(cls + token_ids_a + sep) * [0]
						return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



			def    _lowerCamelCase	(						self       :  Union[str, Any]			,		A       :  str			,		A       :  Optional[str] = None)       ->   Tuple[str]:



						"""simple docstring"""






						_UpperCAmelCase			=       self._tokenizer.model.save(A			,		name=A)
						return tuple(A)



			def    _lowerCamelCase	(						self       :  List[Any]			,		A       :  Tuple			,		A       :  str=None			,		A       :  Union[str, Any]=None			,		A       :  Union[str, Any]=False			,		**A       :  Tuple			,		)       ->   str:



						"""simple docstring"""






						_UpperCAmelCase			=       BertPreTokenizer()
						return super().save_pretrained(A			,		A			,		A			,		A			,		**A)
