






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


SCREAMING_SNAKE_CASE__		:							str 				=				logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__		:							List[Any] 				=				{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__		:							Optional[Any] 				=				{
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

SCREAMING_SNAKE_CASE__		:							Optional[int] 				=				{
    """junnyu/roformer_chinese_small""": 15_36,
    """junnyu/roformer_chinese_base""": 15_36,
    """junnyu/roformer_chinese_char_small""": 5_12,
    """junnyu/roformer_chinese_char_base""": 5_12,
    """junnyu/roformer_small_discriminator""": 1_28,
    """junnyu/roformer_small_generator""": 1_28,
}


SCREAMING_SNAKE_CASE__		:							List[Any] 				=				{
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}







class 				UpperCAmelCase_  (		__lowerCamelCase		):
     __lowerCamelCase      				=  VOCAB_FILES_NAMES
     __lowerCamelCase      				=  PRETRAINED_VOCAB_FILES_MAP
     __lowerCamelCase      				=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __lowerCamelCase      				=  PRETRAINED_INIT_CONFIGURATION
     __lowerCamelCase      				=  RoFormerTokenizer




     def __init__(				self	,						_lowerCAmelCase=None	,						_lowerCAmelCase=None	,						_lowerCAmelCase=True	,						_lowerCAmelCase="[UNK]"	,						_lowerCAmelCase="[SEP]"	,						_lowerCAmelCase="[PAD]"	,						_lowerCAmelCase="[CLS]"	,						_lowerCAmelCase="[MASK]"	,						_lowerCAmelCase=True	,						_lowerCAmelCase=None	,						**_lowerCAmelCase	,						):
          super().__init__(
              _lowerCAmelCase	,						tokenizer_file=_lowerCAmelCase	,						do_lower_case=_lowerCAmelCase	,						unk_token=_lowerCAmelCase	,						sep_token=_lowerCAmelCase	,						pad_token=_lowerCAmelCase	,						cls_token=_lowerCAmelCase	,						mask_token=_lowerCAmelCase	,						tokenize_chinese_chars=_lowerCAmelCase	,						strip_accents=_lowerCAmelCase	,						**_lowerCAmelCase	,						)

          UpperCAmelCase__      :  Union[str, Any]				      =			json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
          if (
              pre_tok_state.get("""lowercase"""	,						_lowerCAmelCase		) != do_lower_case
              or pre_tok_state.get("""strip_accents"""	,						_lowerCAmelCase		) != strip_accents
          ):
               UpperCAmelCase__      :  Union[str, Any]				      =			getattr(_lowerCAmelCase	,						pre_tok_state.pop("""type"""		)		)
               UpperCAmelCase__      :  List[Any]				      =			do_lower_case
               UpperCAmelCase__      :  Union[str, Any]				      =			strip_accents
               UpperCAmelCase__      :  Tuple				      =			pre_tok_class(**_lowerCAmelCase		)

          UpperCAmelCase__      :  Dict				      =			do_lower_case




     def __getstate__(				self		):
          UpperCAmelCase__      :  Any				      =			self.__dict__.copy()
          UpperCAmelCase__      :  int				      =			BertPreTokenizer()
          return state




     def __setstate__(				self	,						_lowerCAmelCase		):
          UpperCAmelCase__      :  List[str]				      =			d
          UpperCAmelCase__      :  str				      =			self.__dict__["""_tokenizer"""].get_vocab()
          UpperCAmelCase__      :  Optional[Any]				      =			PreTokenizer.custom(JiebaPreTokenizer(_lowerCAmelCase		)		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase=None		):
          UpperCAmelCase__      :  List[Any]				      =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase = None		):
          UpperCAmelCase__      :  List[Any]				      =			[self.sep_token_id]
          UpperCAmelCase__      :  Any				      =			[self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep		) * [0]
          return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase = None		):
          UpperCAmelCase__      :  Dict				      =			self._tokenizer.model.save(_lowerCAmelCase	,						name=_lowerCAmelCase		)
          return tuple(_lowerCAmelCase		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase=None	,						_lowerCAmelCase=None	,						_lowerCAmelCase=False	,						**_lowerCAmelCase	,						):
          UpperCAmelCase__      :  List[str]				      =			BertPreTokenizer()
          return super().save_pretrained(_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						_lowerCAmelCase	,						**_lowerCAmelCase		)












"""simple docstring"""




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowerCAmelCase				:     Optional[int]					=				{"""configuration_swin""": ["""SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP""", """SwinConfig""", """SwinOnnxConfig"""]}


try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowerCAmelCase				:     int					=				[
					    """SWIN_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """SwinForImageClassification""",
					    """SwinForMaskedImageModeling""",
					    """SwinModel""",
					    """SwinPreTrainedModel""",
					    """SwinBackbone""",
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowerCAmelCase				:     str					=				[
					    """TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """TFSwinForImageClassification""",
					    """TFSwinForMaskedImageModeling""",
					    """TFSwinModel""",
					    """TFSwinPreTrainedModel""",
					]

if TYPE_CHECKING:
					from .configuration_swin import SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, SwinConfig, SwinOnnxConfig

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_swin import (
										    SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
										    SwinBackbone,
										    SwinForImageClassification,
										    SwinForMaskedImageModeling,
										    SwinModel,
										    SwinPreTrainedModel,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_swin import (
										    TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFSwinForImageClassification,
										    TFSwinForMaskedImageModeling,
										    TFSwinModel,
										    TFSwinPreTrainedModel,
										)

else:
					import sys

					lowerCAmelCase				:     int					=				_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



