

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


__snake_case    :    Optional[Any]         =     logging.get_logger(__name__)

__snake_case    :    str         =     {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__snake_case    :    Optional[int]         =     {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

__snake_case    :    Tuple         =     {
    """google/realm-cc-news-pretrained-embedder""": 5_12,
    """google/realm-cc-news-pretrained-encoder""": 5_12,
    """google/realm-cc-news-pretrained-scorer""": 5_12,
    """google/realm-cc-news-pretrained-openqa""": 5_12,
    """google/realm-orqa-nq-openqa""": 5_12,
    """google/realm-orqa-nq-reader""": 5_12,
    """google/realm-orqa-wq-openqa""": 5_12,
    """google/realm-orqa-wq-reader""": 5_12,
}

__snake_case    :    str         =     {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}


class     __SCREAMING_SNAKE_CASE					(   __lowercase):
						_SCREAMING_SNAKE_CASE	:       Optional[int]   =					VOCAB_FILES_NAMES
						_SCREAMING_SNAKE_CASE	:       Optional[Any]   =					PRETRAINED_VOCAB_FILES_MAP
						_SCREAMING_SNAKE_CASE	:       Optional[Any]   =					PRETRAINED_INIT_CONFIGURATION
						_SCREAMING_SNAKE_CASE	:       Optional[int]   =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_SCREAMING_SNAKE_CASE	:       Optional[Any]   =					RealmTokenizer






						def __init__(	self     ,						_UpperCamelCase=None     ,						_UpperCamelCase=None     ,						_UpperCamelCase=True     ,						_UpperCamelCase="[UNK]"     ,						_UpperCamelCase="[SEP]"     ,						_UpperCamelCase="[PAD]"     ,						_UpperCamelCase="[CLS]"     ,						_UpperCamelCase="[MASK]"     ,						_UpperCamelCase=True     ,						_UpperCamelCase=None     ,						**_UpperCamelCase     ,						):






												"""simple docstring"""
												super().__init__(
												    _UpperCamelCase     ,						tokenizer_file=_UpperCamelCase     ,						do_lower_case=_UpperCamelCase     ,						unk_token=_UpperCamelCase     ,						sep_token=_UpperCamelCase     ,						pad_token=_UpperCamelCase     ,						cls_token=_UpperCamelCase     ,						mask_token=_UpperCamelCase     ,						tokenize_chinese_chars=_UpperCamelCase     ,						strip_accents=_UpperCamelCase     ,						**_UpperCamelCase     ,						)

												lowerCAmelCase__    						=						json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
												if (
												    normalizer_state.get('lowercase'     ,						_UpperCamelCase   ) != do_lower_case
												    or normalizer_state.get('strip_accents'     ,						_UpperCamelCase   ) != strip_accents
												    or normalizer_state.get('handle_chinese_chars'     ,						_UpperCamelCase   ) != tokenize_chinese_chars
												):
																		lowerCAmelCase__    						=						getattr(_UpperCamelCase     ,						normalizer_state.pop('type'   )   )
																		lowerCAmelCase__    						=						do_lower_case
																		lowerCAmelCase__    						=						strip_accents
																		lowerCAmelCase__    						=						tokenize_chinese_chars
																		lowerCAmelCase__    						=						normalizer_class(**_UpperCamelCase   )

												lowerCAmelCase__    						=						do_lower_case






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						**_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						PaddingStrategy.MAX_LENGTH

												lowerCAmelCase__    						=						text
												lowerCAmelCase__    						=						kwargs.pop('text_pair'     ,						_UpperCamelCase   )
												lowerCAmelCase__    						=						kwargs.pop('return_tensors'     ,						_UpperCamelCase   )

												lowerCAmelCase__    						=						{
												    'input_ids': [],
												    'attention_mask': [],
												    'token_type_ids': [],
												}

												for idx, candidate_text in enumerate(_UpperCamelCase   ):
																		if batch_text_pair is not None:
																								lowerCAmelCase__    						=						batch_text_pair[idx]
																		else:
																								lowerCAmelCase__    						=						None

																		lowerCAmelCase__    						=						super().__call__(_UpperCamelCase     ,						_UpperCamelCase     ,						return_tensors=_UpperCamelCase     ,						**_UpperCamelCase   )

																		lowerCAmelCase__    						=						encoded_candidates.get('input_ids'   )
																		lowerCAmelCase__    						=						encoded_candidates.get('attention_mask'   )
																		lowerCAmelCase__    						=						encoded_candidates.get('token_type_ids'   )

																		if encoded_input_ids is not None:
																								output_data["input_ids"].append(_UpperCamelCase   )
																		if encoded_attention_mask is not None:
																								output_data["attention_mask"].append(_UpperCamelCase   )
																		if encoded_token_type_ids is not None:
																								output_data["token_type_ids"].append(_UpperCamelCase   )

												lowerCAmelCase__    						=						{key: item for key, item in output_data.items() if len(_UpperCamelCase   ) != 0}

												return BatchEncoding(_UpperCamelCase     ,						tensor_type=_UpperCamelCase   )






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase=None   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

												return output






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase = None   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						[self.sep_token_id]
												lowerCAmelCase__    						=						[self.cls_token_id]
												if token_ids_a is None:
																		return len(cls + token_ids_a + sep   ) * [0]
												return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]







						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase = None   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self._tokenizer.model.save(_UpperCamelCase     ,						name=_UpperCamelCase   )
												return tuple(_UpperCamelCase   )













from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


__snake_case    :    Tuple         =     {"""configuration_glpn""": ["""GLPN_PRETRAINED_CONFIG_ARCHIVE_MAP""", """GLPNConfig"""]}

try:
					if not is_vision_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__snake_case    :    Dict         =     ["""GLPNFeatureExtractor"""]
					__snake_case    :    Optional[Any]         =     ["""GLPNImageProcessor"""]

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					__snake_case    :    Tuple         =     [
					    """GLPN_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """GLPNForDepthEstimation""",
					    """GLPNLayer""",
					    """GLPNModel""",
					    """GLPNPreTrainedModel""",
					]


if TYPE_CHECKING:
					from .configuration_glpn import GLPN_PRETRAINED_CONFIG_ARCHIVE_MAP, GLPNConfig

					try:
										if not is_vision_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .feature_extraction_glpn import GLPNFeatureExtractor
										from .image_processing_glpn import GLPNImageProcessor

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_glpn import (
										    GLPN_PRETRAINED_MODEL_ARCHIVE_LIST,
										    GLPNForDepthEstimation,
										    GLPNLayer,
										    GLPNModel,
										    GLPNPreTrainedModel,
										)


else:
					import sys

					__snake_case    :    Optional[int]         =     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





