






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


SCREAMING_SNAKE_CASE__		:							str 				=				logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__		:							List[Any] 				=				{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__		:							Optional[Any] 				=				{
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

SCREAMING_SNAKE_CASE__		:							Optional[int] 				=				{
    """junnyu/roformer_chinese_small""": 15_36,
    """junnyu/roformer_chinese_base""": 15_36,
    """junnyu/roformer_chinese_char_small""": 5_12,
    """junnyu/roformer_chinese_char_base""": 5_12,
    """junnyu/roformer_small_discriminator""": 1_28,
    """junnyu/roformer_small_generator""": 1_28,
}


SCREAMING_SNAKE_CASE__		:							List[Any] 				=				{
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}







class 				UpperCAmelCase_  (		__lowerCamelCase		):
     __lowerCamelCase      				=  VOCAB_FILES_NAMES
     __lowerCamelCase      				=  PRETRAINED_VOCAB_FILES_MAP
     __lowerCamelCase      				=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __lowerCamelCase      				=  PRETRAINED_INIT_CONFIGURATION
     __lowerCamelCase      				=  RoFormerTokenizer




     def __init__(				self	,						_lowerCAmelCase=None	,						_lowerCAmelCase=None	,						_lowerCAmelCase=True	,						_lowerCAmelCase="[UNK]"	,						_lowerCAmelCase="[SEP]"	,						_lowerCAmelCase="[PAD]"	,						_lowerCAmelCase="[CLS]"	,						_lowerCAmelCase="[MASK]"	,						_lowerCAmelCase=True	,						_lowerCAmelCase=None	,						**_lowerCAmelCase	,						):
          super().__init__(
              _lowerCAmelCase	,						tokenizer_file=_lowerCAmelCase	,						do_lower_case=_lowerCAmelCase	,						unk_token=_lowerCAmelCase	,						sep_token=_lowerCAmelCase	,						pad_token=_lowerCAmelCase	,						cls_token=_lowerCAmelCase	,						mask_token=_lowerCAmelCase	,						tokenize_chinese_chars=_lowerCAmelCase	,						strip_accents=_lowerCAmelCase	,						**_lowerCAmelCase	,						)

          UpperCAmelCase__      :  Union[str, Any]				      =			json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
          if (
              pre_tok_state.get("""lowercase"""	,						_lowerCAmelCase		) != do_lower_case
              or pre_tok_state.get("""strip_accents"""	,						_lowerCAmelCase		) != strip_accents
          ):
               UpperCAmelCase__      :  Union[str, Any]				      =			getattr(_lowerCAmelCase	,						pre_tok_state.pop("""type"""		)		)
               UpperCAmelCase__      :  List[Any]				      =			do_lower_case
               UpperCAmelCase__      :  Union[str, Any]				      =			strip_accents
               UpperCAmelCase__      :  Tuple				      =			pre_tok_class(**_lowerCAmelCase		)

          UpperCAmelCase__      :  Dict				      =			do_lower_case




     def __getstate__(				self		):
          UpperCAmelCase__      :  Any				      =			self.__dict__.copy()
          UpperCAmelCase__      :  int				      =			BertPreTokenizer()
          return state




     def __setstate__(				self	,						_lowerCAmelCase		):
          UpperCAmelCase__      :  List[str]				      =			d
          UpperCAmelCase__      :  str				      =			self.__dict__["""_tokenizer"""].get_vocab()
          UpperCAmelCase__      :  Optional[Any]				      =			PreTokenizer.custom(JiebaPreTokenizer(_lowerCAmelCase		)		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase=None		):
          UpperCAmelCase__      :  List[Any]				      =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase = None		):
          UpperCAmelCase__      :  List[Any]				      =			[self.sep_token_id]
          UpperCAmelCase__      :  Any				      =			[self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep		) * [0]
          return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase = None		):
          UpperCAmelCase__      :  Dict				      =			self._tokenizer.model.save(_lowerCAmelCase	,						name=_lowerCAmelCase		)
          return tuple(_lowerCAmelCase		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase=None	,						_lowerCAmelCase=None	,						_lowerCAmelCase=False	,						**_lowerCAmelCase	,						):
          UpperCAmelCase__      :  List[str]				      =			BertPreTokenizer()
          return super().save_pretrained(_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						**_lowerCAmelCase		)













'''simple docstring'''






from ..utils import DummyObject, requires_backends




class 					UpperCAmelCase	(     metaclass=snake_case_  ):
      _lowercase:     List[Any]			=  ['''torch''', '''scipy''']




      def __init__(     self     :					Tuple     ,     *__snake_case     :					Dict     ,     **__snake_case     :					List[str]    )							-> str:
            requires_backends(self     ,     ["""torch""", """scipy"""]    )




      @classmethod
      def 							lowercase__       (     cls     :					str     ,     *__snake_case     :					List[str]     ,     **__snake_case     :					str    )							-> Tuple:
            requires_backends(cls     ,     ["""torch""", """scipy"""]    )




      @classmethod
      def 							lowercase__       (     cls     :					Optional[int]     ,     *__snake_case     :					int     ,     **__snake_case     :					Union[str, Any]    )							-> Optional[int]:
            requires_backends(cls     ,     ["""torch""", """scipy"""]    )






