






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


SCREAMING_SNAKE_CASE__		:							Dict 				=				logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__		:							int 				=				{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__		:							List[str] 				=				{
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

SCREAMING_SNAKE_CASE__		:							Optional[Any] 				=				{
    """google/realm-cc-news-pretrained-embedder""": 5_12,
    """google/realm-cc-news-pretrained-encoder""": 5_12,
    """google/realm-cc-news-pretrained-scorer""": 5_12,
    """google/realm-cc-news-pretrained-openqa""": 5_12,
    """google/realm-orqa-nq-openqa""": 5_12,
    """google/realm-orqa-nq-reader""": 5_12,
    """google/realm-orqa-wq-openqa""": 5_12,
    """google/realm-orqa-wq-reader""": 5_12,
}

SCREAMING_SNAKE_CASE__		:							Optional[Any] 				=				{
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}







class 				UpperCAmelCase_  (		__lowerCamelCase		):
     __lowerCamelCase      				=  VOCAB_FILES_NAMES
     __lowerCamelCase      				=  PRETRAINED_VOCAB_FILES_MAP
     __lowerCamelCase      				=  PRETRAINED_INIT_CONFIGURATION
     __lowerCamelCase      				=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __lowerCamelCase      				=  RealmTokenizer




     def __init__(				self	,						_lowerCAmelCase=None	,						_lowerCAmelCase=None	,						_lowerCAmelCase=True	,						_lowerCAmelCase="[UNK]"	,						_lowerCAmelCase="[SEP]"	,						_lowerCAmelCase="[PAD]"	,						_lowerCAmelCase="[CLS]"	,						_lowerCAmelCase="[MASK]"	,						_lowerCAmelCase=True	,						_lowerCAmelCase=None	,						**_lowerCAmelCase	,						):
          super().__init__(
              _lowerCAmelCase	,						tokenizer_file=_lowerCAmelCase	,						do_lower_case=_lowerCAmelCase	,						unk_token=_lowerCAmelCase	,						sep_token=_lowerCAmelCase	,						pad_token=_lowerCAmelCase	,						cls_token=_lowerCAmelCase	,						mask_token=_lowerCAmelCase	,						tokenize_chinese_chars=_lowerCAmelCase	,						strip_accents=_lowerCAmelCase	,						**_lowerCAmelCase	,						)

          UpperCAmelCase__      :  Tuple				      =			json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
          if (
              normalizer_state.get("""lowercase"""	,						_lowerCAmelCase		) != do_lower_case
              or normalizer_state.get("""strip_accents"""	,						_lowerCAmelCase		) != strip_accents
              or normalizer_state.get("""handle_chinese_chars"""	,						_lowerCAmelCase		) != tokenize_chinese_chars
          ):
               UpperCAmelCase__      :  Any				      =			getattr(_lowerCAmelCase	,						normalizer_state.pop("""type"""		)		)
               UpperCAmelCase__      :  str				      =			do_lower_case
               UpperCAmelCase__      :  Tuple				      =			strip_accents
               UpperCAmelCase__      :  Tuple				      =			tokenize_chinese_chars
               UpperCAmelCase__      :  Union[str, Any]				      =			normalizer_class(**_lowerCAmelCase		)

          UpperCAmelCase__      :  Dict				      =			do_lower_case




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						**_lowerCAmelCase		):
          UpperCAmelCase__      :  List[Any]				      =			PaddingStrategy.MAX_LENGTH

          UpperCAmelCase__      :  Optional[int]				      =			text
          UpperCAmelCase__      :  Optional[int]				      =			kwargs.pop("""text_pair"""	,						_lowerCAmelCase		)
          UpperCAmelCase__      :  Optional[int]				      =			kwargs.pop("""return_tensors"""	,						_lowerCAmelCase		)

          UpperCAmelCase__      :  Optional[Any]				      =			{
              """input_ids""": [],
              """attention_mask""": [],
              """token_type_ids""": [],
          }

          for idx, candidate_text in enumerate(_lowerCAmelCase		):
               if batch_text_pair is not None:
                    UpperCAmelCase__      :  str				      =			batch_text_pair[idx]
               else:
                    UpperCAmelCase__      :  Any				      =			None

               UpperCAmelCase__      :  str				      =			super().__call__(_lowerCAmelCase	,						_lowerCAmelCase	,						return_tensors=_lowerCAmelCase	,						**_lowerCAmelCase		)

               UpperCAmelCase__      :  Union[str, Any]				      =			encoded_candidates.get("""input_ids"""		)
               UpperCAmelCase__      :  str				      =			encoded_candidates.get("""attention_mask"""		)
               UpperCAmelCase__      :  Union[str, Any]				      =			encoded_candidates.get("""token_type_ids"""		)

               if encoded_input_ids is not None:
                    output_data["input_ids"].append(_lowerCAmelCase		)
               if encoded_attention_mask is not None:
                    output_data["attention_mask"].append(_lowerCAmelCase		)
               if encoded_token_type_ids is not None:
                    output_data["token_type_ids"].append(_lowerCAmelCase		)

          UpperCAmelCase__      :  Union[str, Any]				      =			{key: item for key, item in output_data.items() if len(_lowerCAmelCase		) != 0}

          return BatchEncoding(_lowerCAmelCase	,						tensor_type=_lowerCAmelCase		)




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase=None		):
          UpperCAmelCase__      :  List[Any]				      =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase = None		):
          UpperCAmelCase__      :  Any				      =			[self.sep_token_id]
          UpperCAmelCase__      :  int				      =			[self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep		) * [0]
          return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]




     def   __UpperCAmelCase  (				self	,						_lowerCAmelCase	,						_lowerCAmelCase = None		):
          UpperCAmelCase__      :  List[str]				      =			self._tokenizer.model.save(_lowerCAmelCase	,						name=_lowerCAmelCase		)
          return tuple(_lowerCAmelCase		)











from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_lowerCAmelCase	:   Any    =   {
    "configuration_chinese_clip": [
        "CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "ChineseCLIPConfig",
        "ChineseCLIPOnnxConfig",
        "ChineseCLIPTextConfig",
        "ChineseCLIPVisionConfig",
    ],
    "processing_chinese_clip": ["ChineseCLIPProcessor"],
}

try:
	if not is_vision_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowerCAmelCase	:   str    =   ["ChineseCLIPFeatureExtractor"]
	_lowerCAmelCase	:   Any    =   ["ChineseCLIPImageProcessor"]

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowerCAmelCase	:   Optional[int]    =   [
	    "CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST",
	    "ChineseCLIPModel",
	    "ChineseCLIPPreTrainedModel",
	    "ChineseCLIPTextModel",
	    "ChineseCLIPVisionModel",
	]

if TYPE_CHECKING:
	from .configuration_chinese_clip import (
	    CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    ChineseCLIPConfig,
	    ChineseCLIPOnnxConfig,
	    ChineseCLIPTextConfig,
	    ChineseCLIPVisionConfig,
	)
	from .processing_chinese_clip import ChineseCLIPProcessor

	try:
		if not is_vision_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .feature_extraction_chinese_clip import ChineseCLIPFeatureExtractor, ChineseCLIPImageProcessor

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_chinese_clip import (
		    CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
		    ChineseCLIPModel,
		    ChineseCLIPPreTrainedModel,
		    ChineseCLIPTextModel,
		    ChineseCLIPVisionModel,
		)

else:
	import sys

	_lowerCAmelCase	:   Any    =   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
