




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


UpperCamelCase__					    =    logging.get_logger(__name__)

UpperCamelCase__					    =    {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase__					    =    {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCamelCase__					    =    {
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

UpperCamelCase__					    =    {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}
class     a__					(      snake_case__   ):
						_a							:					List[Any]      =					VOCAB_FILES_NAMES
						_a							:					List[str]      =					PRETRAINED_VOCAB_FILES_MAP
						_a							:					List[Any]      =					PRETRAINED_INIT_CONFIGURATION
						_a							:					List[Any]      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_a							:					List[str]      =					RealmTokenizer






						def __init__(     self	,    _A=None	,    _A=None	,    _A=True	,    _A="[UNK]"	,    _A="[SEP]"	,    _A="[PAD]"	,    _A="[CLS]"	,    _A="[MASK]"	,    _A=True	,    _A=None	,    **_A	,    ):




											"""simple docstring"""



											super().__init__(
											    _A	,    tokenizer_file=_A	,    do_lower_case=_A	,    unk_token=_A	,    sep_token=_A	,    pad_token=_A	,    cls_token=_A	,    mask_token=_A	,    tokenize_chinese_chars=_A	,    strip_accents=_A	,    **_A	,    )

											__lowerCAmelCase            = json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
											if (
											    normalizer_state.get("lowercase"	,    _A    ) != do_lower_case
											    or normalizer_state.get("strip_accents"	,    _A    ) != strip_accents
											    or normalizer_state.get("handle_chinese_chars"	,    _A    ) != tokenize_chinese_chars
											):
																__lowerCAmelCase            = getattr(_A	,    normalizer_state.pop("type"    )    )
																__lowerCAmelCase            = do_lower_case
																__lowerCAmelCase            = strip_accents
																__lowerCAmelCase            = tokenize_chinese_chars
																__lowerCAmelCase            = normalizer_class(**_A    )

											__lowerCAmelCase            = do_lower_case






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    **_A    ):




											"""simple docstring"""



											__lowerCAmelCase            = PaddingStrategy.MAX_LENGTH

											__lowerCAmelCase            = text
											__lowerCAmelCase            = kwargs.pop("text_pair"	,    _A    )
											__lowerCAmelCase            = kwargs.pop("return_tensors"	,    _A    )

											__lowerCAmelCase            = {
											    "input_ids": [],
											    "attention_mask": [],
											    "token_type_ids": [],
											}

											for idx, candidate_text in enumerate(_A    ):
																if batch_text_pair is not None:
																					__lowerCAmelCase            = batch_text_pair[idx]
																else:
																					__lowerCAmelCase            = None

																__lowerCAmelCase            = super().__call__(_A	,    _A	,    return_tensors=_A	,    **_A    )

																__lowerCAmelCase            = encoded_candidates.get("input_ids"    )
																__lowerCAmelCase            = encoded_candidates.get("attention_mask"    )
																__lowerCAmelCase            = encoded_candidates.get("token_type_ids"    )

																if encoded_input_ids is not None:
																					output_data["input_ids"].append(_A    )
																if encoded_attention_mask is not None:
																					output_data["attention_mask"].append(_A    )
																if encoded_token_type_ids is not None:
																					output_data["token_type_ids"].append(_A    )

											__lowerCAmelCase            = {key: item for key, item in output_data.items() if len(_A    ) != 0}

											return BatchEncoding(_A	,    tensor_type=_A    )






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    _A=None    ):




											"""simple docstring"""



											__lowerCAmelCase            = [self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

											return output






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    _A = None    ):




											"""simple docstring"""



											__lowerCAmelCase            = [self.sep_token_id]
											__lowerCAmelCase            = [self.cls_token_id]
											if token_ids_a is None:
																return len(cls + token_ids_a + sep    ) * [0]
											return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    _A = None    ):




											"""simple docstring"""



											__lowerCAmelCase            = self._tokenizer.model.save(_A	,    name=_A    )
											return tuple(_A    )










from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


UpperCamelCase__					    =    {
    """configuration_ctrl""": ["""CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CTRLConfig"""],
    """tokenization_ctrl""": ["""CTRLTokenizer"""],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__					    =    [
					    """CTRL_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """CTRLForSequenceClassification""",
					    """CTRLLMHeadModel""",
					    """CTRLModel""",
					    """CTRLPreTrainedModel""",
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__					    =    [
					    """TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """TFCTRLForSequenceClassification""",
					    """TFCTRLLMHeadModel""",
					    """TFCTRLModel""",
					    """TFCTRLPreTrainedModel""",
					]


if TYPE_CHECKING:
					from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
					from .tokenization_ctrl import CTRLTokenizer

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_ctrl import (
										    CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
										    CTRLForSequenceClassification,
										    CTRLLMHeadModel,
										    CTRLModel,
										    CTRLPreTrainedModel,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_ctrl import (
										    TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFCTRLForSequenceClassification,
										    TFCTRLLMHeadModel,
										    TFCTRLModel,
										    TFCTRLPreTrainedModel,
										)

else:
					import sys

					UpperCamelCase__					    =    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




