


"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCAmelCase				=						logging.get_logger(__name__)

UpperCAmelCase				=						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase				=						{
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

UpperCAmelCase				=						{
    """junnyu/roformer_chinese_small""": 1536,
    """junnyu/roformer_chinese_base""": 1536,
    """junnyu/roformer_chinese_char_small""": 512,
    """junnyu/roformer_chinese_char_base""": 512,
    """junnyu/roformer_small_discriminator""": 128,
    """junnyu/roformer_small_generator""": 128,
}


UpperCAmelCase				=						{
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}




class 	lowercase__				(					A_				):
   __UpperCAmelCase							       =					VOCAB_FILES_NAMES
   __UpperCAmelCase							       =					PRETRAINED_VOCAB_FILES_MAP
   __UpperCAmelCase							       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCAmelCase							       =					PRETRAINED_INIT_CONFIGURATION
   __UpperCAmelCase							       =					RoFormerTokenizer





   def __init__(							self      ,   SCREAMING_SNAKE_CASE=None      ,   SCREAMING_SNAKE_CASE=None      ,   SCREAMING_SNAKE_CASE=True      ,   SCREAMING_SNAKE_CASE="[UNK]"      ,   SCREAMING_SNAKE_CASE="[SEP]"      ,   SCREAMING_SNAKE_CASE="[PAD]"      ,   SCREAMING_SNAKE_CASE="[CLS]"      ,   SCREAMING_SNAKE_CASE="[MASK]"      ,   SCREAMING_SNAKE_CASE=True      ,   SCREAMING_SNAKE_CASE=None      ,   **SCREAMING_SNAKE_CASE      ,   )  ->  Union[str, Any]:
      super().__init__(
          SCREAMING_SNAKE_CASE      ,   tokenizer_file=SCREAMING_SNAKE_CASE      ,   do_lower_case=SCREAMING_SNAKE_CASE      ,   unk_token=SCREAMING_SNAKE_CASE      ,   sep_token=SCREAMING_SNAKE_CASE      ,   pad_token=SCREAMING_SNAKE_CASE      ,   cls_token=SCREAMING_SNAKE_CASE      ,   mask_token=SCREAMING_SNAKE_CASE      ,   tokenize_chinese_chars=SCREAMING_SNAKE_CASE      ,   strip_accents=SCREAMING_SNAKE_CASE      ,   **SCREAMING_SNAKE_CASE      ,   )

      _lowerCamelCase	:					Tuple   =		json.loads(self.backend_tokenizer.normalizer.__getstate__())
      if (
          pre_tok_state.get("""lowercase"""      ,   SCREAMING_SNAKE_CASE) != do_lower_case
          or pre_tok_state.get("""strip_accents"""      ,   SCREAMING_SNAKE_CASE) != strip_accents
      ):
         _lowerCamelCase	:					str   =		getattr(SCREAMING_SNAKE_CASE      ,   pre_tok_state.pop("""type"""))
         _lowerCamelCase	:					List[str]   =		do_lower_case
         _lowerCamelCase	:					str   =		strip_accents
         _lowerCamelCase	:					Optional[Any]   =		pre_tok_class(**SCREAMING_SNAKE_CASE)

      _lowerCamelCase	:					Any   =		do_lower_case





   def __getstate__(							self)  ->  str:
      _lowerCamelCase	:					Any   =		self.__dict__.copy()
      _lowerCamelCase	:					Dict   =		BertPreTokenizer()
      return state





   def __setstate__(							self      ,   SCREAMING_SNAKE_CASE)  ->  Union[str, Any]:
      _lowerCamelCase	:					Tuple   =		d
      _lowerCamelCase	:					List[Any]   =		self.__dict__["""_tokenizer"""].get_vocab()
      _lowerCamelCase	:					Optional[int]   =		PreTokenizer.custom(JiebaPreTokenizer(SCREAMING_SNAKE_CASE))





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE=None)  ->  Any:
      _lowerCamelCase	:					List[Any]   =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

      return output





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE = None)  ->  List[int]:
      _lowerCamelCase	:					List[Any]   =		[self.sep_token_id]
      _lowerCamelCase	:					Optional[Any]   =		[self.cls_token_id]
      if token_ids_a is None:
         return len(cls + token_ids_a + sep) * [0]
      return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE = None)  ->  Tuple[str]:
      _lowerCamelCase	:					Dict   =		self._tokenizer.model.save(SCREAMING_SNAKE_CASE      ,   name=SCREAMING_SNAKE_CASE)
      return tuple(SCREAMING_SNAKE_CASE)





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE=None      ,   SCREAMING_SNAKE_CASE=None      ,   SCREAMING_SNAKE_CASE=False      ,   **SCREAMING_SNAKE_CASE      ,   )  ->  Union[str, Any]:
      _lowerCamelCase	:					List[Any]   =		BertPreTokenizer()
      return super().save_pretrained(SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   SCREAMING_SNAKE_CASE      ,   **SCREAMING_SNAKE_CASE)







import warnings

from ...utils import logging
from .image_processing_chinese_clip import ChineseCLIPImageProcessor


UpperCamelCase__ :   Dict						=							logging.get_logger(__name__)







class 				lowerCAmelCase_       (					lowerCamelCase_							):






				def __init__(			self      ,*snake_case__      ,**snake_case__ ):
							warnings.warn(
							    'The class ChineseCLIPFeatureExtractor is deprecated and will be removed in version 5 of Transformers.'
							    ' Please use ChineseCLIPImageProcessor instead.'      ,snake_case__      ,)
							super().__init__(*snake_case__      ,**snake_case__ )



