


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCAmelCase   :    Optional[Any]     =		logging.get_logger(__name__)

UpperCAmelCase   :    Union[str, Any]     =		{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase   :    List[Any]     =		{
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

UpperCAmelCase   :    Tuple     =		{
    "junnyu/roformer_chinese_small": 15_36,
    "junnyu/roformer_chinese_base": 15_36,
    "junnyu/roformer_chinese_char_small": 5_12,
    "junnyu/roformer_chinese_char_base": 5_12,
    "junnyu/roformer_small_discriminator": 1_28,
    "junnyu/roformer_small_generator": 1_28,
}


UpperCAmelCase   :    Any     =		{
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}







class        __lowercase       (							a_  ):



		"""simple docstring"""

		UpperCamelCase     :					Union[str, Any]			       =		VOCAB_FILES_NAMES
		UpperCamelCase     :					Tuple			       =		PRETRAINED_VOCAB_FILES_MAP
		UpperCamelCase     :					Dict			       =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		UpperCamelCase     :					Dict			       =		PRETRAINED_INIT_CONFIGURATION
		UpperCamelCase     :					List[Any]			       =		RoFormerTokenizer







		def __init__( self		,      A=None		,      A=None		,      A=True		,      A="[UNK]"		,      A="[SEP]"		,      A="[PAD]"		,      A="[CLS]"		,      A="[MASK]"		,      A=True		,      A=None		,      **A		,      )  -> Dict:







					'''simple docstring'''







					super().__init__(
					    A		,      tokenizer_file=A		,      do_lower_case=A		,      unk_token=A		,      sep_token=A		,      pad_token=A		,      cls_token=A		,      mask_token=A		,      tokenize_chinese_chars=A		,      strip_accents=A		,      **A		,      )

					lowerCamelCase    		=  json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
					if (
					    pre_tok_state.get("""lowercase"""		,      A	) != do_lower_case
					    or pre_tok_state.get("""strip_accents"""		,      A	) != strip_accents
					):
								lowerCamelCase    		=  getattr(A		,      pre_tok_state.pop("""type"""	)	)
								lowerCamelCase    		=  do_lower_case
								lowerCamelCase    		=  strip_accents
								lowerCamelCase    		=  pre_tok_class(**A	)

					lowerCamelCase    		=  do_lower_case







		def __getstate__( self	)  -> Tuple:







					'''simple docstring'''







					lowerCamelCase    		=  self.__dict__.copy()
					lowerCamelCase    		=  BertPreTokenizer()
					return state







		def __setstate__( self		,      A	)  -> Tuple:







					'''simple docstring'''







					lowerCamelCase    		=  d
					lowerCamelCase    		=  self.__dict__["""_tokenizer"""].get_vocab()
					lowerCamelCase    		=  PreTokenizer.custom(JiebaPreTokenizer(A	)	)







		def 					__A   ( self		,      A		,      A=None	)  -> Tuple:







					'''simple docstring'''







					lowerCamelCase    		=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
								output += token_ids_a + [self.sep_token_id]

					return output







		def 					__A   ( self		,      A		,      A = None	)  -> List[int]:







					'''simple docstring'''







					lowerCamelCase    		=  [self.sep_token_id]
					lowerCamelCase    		=  [self.cls_token_id]
					if token_ids_a is None:
								return len(cls + token_ids_a + sep	) * [0]
					return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]







		def 					__A   ( self		,      A		,      A = None	)  -> Tuple[str]:







					'''simple docstring'''







					lowerCamelCase    		=  self._tokenizer.model.save(A		,      name=A	)
					return tuple(A	)







		def 					__A   ( self		,      A		,      A=None		,      A=None		,      A=False		,      **A		,      )  -> int:







					'''simple docstring'''







					lowerCamelCase    		=  BertPreTokenizer()
					return super().save_pretrained(A		,      A		,      A		,      A		,      **A	)





from ..utils import DummyObject, requires_backends







class        __lowercase       (							metaclass=a_  ):



		"""simple docstring"""

		UpperCamelCase     :					Tuple			       =		["flax"]







		def __init__( self		,      *A		,      **A	)  -> str:







					'''simple docstring'''







					requires_backends(self		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> int:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> str:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







class        __lowercase       (							metaclass=a_  ):



		"""simple docstring"""

		UpperCamelCase     :					Optional[Any]			       =		["flax"]







		def __init__( self		,      *A		,      **A	)  -> Dict:







					'''simple docstring'''







					requires_backends(self		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> List[str]:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Dict:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







class        __lowercase       (							metaclass=a_  ):



		"""simple docstring"""

		UpperCamelCase     :					str			       =		["flax"]







		def __init__( self		,      *A		,      **A	)  -> Dict:







					'''simple docstring'''







					requires_backends(self		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Union[str, Any]:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Optional[int]:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







class        __lowercase       (							metaclass=a_  ):



		"""simple docstring"""

		UpperCamelCase     :					str			       =		["flax"]







		def __init__( self		,      *A		,      **A	)  -> List[Any]:







					'''simple docstring'''







					requires_backends(self		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Any:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Union[str, Any]:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







class        __lowercase       (							metaclass=a_  ):



		"""simple docstring"""

		UpperCamelCase     :					Dict			       =		["flax"]







		def __init__( self		,      *A		,      **A	)  -> Optional[Any]:







					'''simple docstring'''







					requires_backends(self		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Optional[Any]:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Any:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







class        __lowercase       (							metaclass=a_  ):



		"""simple docstring"""

		UpperCamelCase     :					Optional[int]			       =		["flax"]







		def __init__( self		,      *A		,      **A	)  -> int:







					'''simple docstring'''







					requires_backends(self		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> int:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> str:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







class        __lowercase       (							metaclass=a_  ):



		"""simple docstring"""

		UpperCamelCase     :					Union[str, Any]			       =		["flax"]







		def __init__( self		,      *A		,      **A	)  -> List[Any]:







					'''simple docstring'''







					requires_backends(self		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Tuple:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Optional[int]:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







class        __lowercase       (							metaclass=a_  ):



		"""simple docstring"""

		UpperCamelCase     :					Optional[Any]			       =		["flax"]







		def __init__( self		,      *A		,      **A	)  -> Optional[Any]:







					'''simple docstring'''







					requires_backends(self		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> int:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> int:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







class        __lowercase       (							metaclass=a_  ):



		"""simple docstring"""

		UpperCamelCase     :					Tuple			       =		["flax"]







		def __init__( self		,      *A		,      **A	)  -> str:







					'''simple docstring'''







					requires_backends(self		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> List[Any]:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> str:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







class        __lowercase       (							metaclass=a_  ):



		"""simple docstring"""

		UpperCamelCase     :					Any			       =		["flax"]







		def __init__( self		,      *A		,      **A	)  -> List[str]:







					'''simple docstring'''







					requires_backends(self		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> int:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> str:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







class        __lowercase       (							metaclass=a_  ):



		"""simple docstring"""

		UpperCamelCase     :					str			       =		["flax"]







		def __init__( self		,      *A		,      **A	)  -> Dict:







					'''simple docstring'''







					requires_backends(self		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Optional[int]:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Dict:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







class        __lowercase       (							metaclass=a_  ):



		"""simple docstring"""

		UpperCamelCase     :					Optional[int]			       =		["flax"]







		def __init__( self		,      *A		,      **A	)  -> List[str]:







					'''simple docstring'''







					requires_backends(self		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Any:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Any:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







class        __lowercase       (							metaclass=a_  ):



		"""simple docstring"""

		UpperCamelCase     :					Tuple			       =		["flax"]







		def __init__( self		,      *A		,      **A	)  -> str:







					'''simple docstring'''







					requires_backends(self		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Dict:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)







		@classmethod
		def 					__A   ( cls		,      *A		,      **A	)  -> Union[str, Any]:







					'''simple docstring'''







					requires_backends(cls		,      ["""flax"""]	)

