





'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowerCAmelCase       		=      logging.get_logger(__name__)

lowerCAmelCase       		=      {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase       		=      {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCAmelCase       		=      {
    """google/realm-cc-news-pretrained-embedder""": 5_12,
    """google/realm-cc-news-pretrained-encoder""": 5_12,
    """google/realm-cc-news-pretrained-scorer""": 5_12,
    """google/realm-cc-news-pretrained-openqa""": 5_12,
    """google/realm-orqa-nq-openqa""": 5_12,
    """google/realm-orqa-nq-reader""": 5_12,
    """google/realm-orqa-wq-openqa""": 5_12,
    """google/realm-orqa-wq-reader""": 5_12,
}

lowerCAmelCase       		=      {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}
class    lowerCamelCase  (      A_       ):
	snake_case_      =   VOCAB_FILES_NAMES
	snake_case_      =   PRETRAINED_VOCAB_FILES_MAP
	snake_case_      =   PRETRAINED_INIT_CONFIGURATION
	snake_case_      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case_      =   RealmTokenizer



	def __init__(   self		,      a_=None		,      a_=None		,      a_=True		,      a_="[UNK]"		,      a_="[SEP]"		,      a_="[PAD]"		,      a_="[CLS]"		,      a_="[MASK]"		,      a_=True		,      a_=None		,      **a_		,      ):
			super().__init__(
			    a_		,      tokenizer_file=a_		,      do_lower_case=a_		,      unk_token=a_		,      sep_token=a_		,      pad_token=a_		,      cls_token=a_		,      mask_token=a_		,      tokenize_chinese_chars=a_		,      strip_accents=a_		,      **a_		,      )

			lowerCAmelCase     :    List[str]						=   json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
			if (
			    normalizer_state.get("lowercase"		,      a_	) != do_lower_case
			    or normalizer_state.get("strip_accents"		,      a_	) != strip_accents
			    or normalizer_state.get("handle_chinese_chars"		,      a_	) != tokenize_chinese_chars
			):
					lowerCAmelCase     :    int						=   getattr(a_		,      normalizer_state.pop("type"	)	)
					lowerCAmelCase     :    Optional[Any]						=   do_lower_case
					lowerCAmelCase     :    List[str]						=   strip_accents
					lowerCAmelCase     :    Dict						=   tokenize_chinese_chars
					lowerCAmelCase     :    Optional[Any]						=   normalizer_class(**a_	)

			lowerCAmelCase     :    Any						=   do_lower_case



	def   _lowerCamelCase    (   self		,      a_		,      **a_	):
			lowerCAmelCase     :    int						=   PaddingStrategy.MAX_LENGTH

			lowerCAmelCase     :    Optional[int]						=   text
			lowerCAmelCase     :    Tuple						=   kwargs.pop("text_pair"		,      a_	)
			lowerCAmelCase     :    List[Any]						=   kwargs.pop("return_tensors"		,      a_	)

			lowerCAmelCase     :    Tuple						=   {
			    """input_ids""": [],
			    """attention_mask""": [],
			    """token_type_ids""": [],
			}

			for idx, candidate_text in enumerate(a_	):
					if batch_text_pair is not None:
							lowerCAmelCase     :    List[Any]						=   batch_text_pair[idx]
					else:
							lowerCAmelCase     :    Any						=   None

					lowerCAmelCase     :    List[str]						=   super().__call__(a_		,      a_		,      return_tensors=a_		,      **a_	)

					lowerCAmelCase     :    str						=   encoded_candidates.get("input_ids"	)
					lowerCAmelCase     :    Union[str, Any]						=   encoded_candidates.get("attention_mask"	)
					lowerCAmelCase     :    int						=   encoded_candidates.get("token_type_ids"	)

					if encoded_input_ids is not None:
							output_data["input_ids"].append(a_	)
					if encoded_attention_mask is not None:
							output_data["attention_mask"].append(a_	)
					if encoded_token_type_ids is not None:
							output_data["token_type_ids"].append(a_	)

			lowerCAmelCase     :    List[str]						=   {key: item for key, item in output_data.items() if len(a_	) != 0}

			return BatchEncoding(a_		,      tensor_type=a_	)



	def   _lowerCamelCase    (   self		,      a_		,      a_=None	):
			lowerCAmelCase     :    Dict						=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

			if token_ids_a:
					output += token_ids_a + [self.sep_token_id]

			return output



	def   _lowerCamelCase    (   self		,      a_		,      a_ = None	):
			lowerCAmelCase     :    List[str]						=   [self.sep_token_id]
			lowerCAmelCase     :    Union[str, Any]						=   [self.cls_token_id]
			if token_ids_a is None:
					return len(cls + token_ids_a + sep	) * [0]
			return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]





	def   _lowerCamelCase    (   self		,      a_		,      a_ = None	):
			lowerCAmelCase     :    Any						=   self._tokenizer.model.save(a_		,      name=a_	)
			return tuple(a_	)










'''simple docstring'''






import warnings

from ...utils import logging
from .image_processing_chinese_clip import ChineseCLIPImageProcessor


lowerCAmelCase       		=      logging.get_logger(__name__)
class 		lowerCamelCase  (      _A       ):



	def __init__(   self		,      *a_		,      **a_	):
			warnings.warn(
			    "The class ChineseCLIPFeatureExtractor is deprecated and will be removed in version 5 of Transformers."
			    " Please use ChineseCLIPImageProcessor instead."		,      a_		,      )
			super().__init__(*a_		,      **a_	)



