




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


SCREAMING_SNAKE_CASE__     :     Optional[int]		    =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     :     Dict		    =       {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE__     :     int		    =       {
    "vocab_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"
        ),
        "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt",
        "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-openqa": (
            "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-reader": (
            "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-openqa": (
            "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-reader": (
            "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"
        ),
    },
}

SCREAMING_SNAKE_CASE__     :     Optional[int]		    =       {
    "google/realm-cc-news-pretrained-embedder": 5_12,
    "google/realm-cc-news-pretrained-encoder": 5_12,
    "google/realm-cc-news-pretrained-scorer": 5_12,
    "google/realm-cc-news-pretrained-openqa": 5_12,
    "google/realm-orqa-nq-openqa": 5_12,
    "google/realm-orqa-nq-reader": 5_12,
    "google/realm-orqa-wq-openqa": 5_12,
    "google/realm-orqa-wq-reader": 5_12,
}

SCREAMING_SNAKE_CASE__     :     List[str]		    =       {
    "google/realm-cc-news-pretrained-embedder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-encoder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-scorer": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-reader": {"do_lower_case": True},
    "google/realm-orqa-wq-openqa": {"do_lower_case": True},
    "google/realm-orqa-wq-reader": {"do_lower_case": True},
}



class       a_       (     SCREAMING_SNAKE_CASE__     ):
	A    							= VOCAB_FILES_NAMES
	A    							= PRETRAINED_VOCAB_FILES_MAP
	A    							= PRETRAINED_INIT_CONFIGURATION
	A    							= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A    							= RealmTokenizer





	def __init__(							self      ,     SCREAMING_SNAKE_CASE=None      ,     SCREAMING_SNAKE_CASE=None      ,     SCREAMING_SNAKE_CASE=True      ,     SCREAMING_SNAKE_CASE="[UNK]"      ,     SCREAMING_SNAKE_CASE="[SEP]"      ,     SCREAMING_SNAKE_CASE="[PAD]"      ,     SCREAMING_SNAKE_CASE="[CLS]"      ,     SCREAMING_SNAKE_CASE="[MASK]"      ,     SCREAMING_SNAKE_CASE=True      ,     SCREAMING_SNAKE_CASE=None      ,     **SCREAMING_SNAKE_CASE      ,     )  ->					Optional[Any]:




				"""simple docstring"""



				super().__init__(
				    SCREAMING_SNAKE_CASE      ,     tokenizer_file=SCREAMING_SNAKE_CASE      ,     do_lower_case=SCREAMING_SNAKE_CASE      ,     unk_token=SCREAMING_SNAKE_CASE      ,     sep_token=SCREAMING_SNAKE_CASE      ,     pad_token=SCREAMING_SNAKE_CASE      ,     cls_token=SCREAMING_SNAKE_CASE      ,     mask_token=SCREAMING_SNAKE_CASE      ,     tokenize_chinese_chars=SCREAMING_SNAKE_CASE      ,     strip_accents=SCREAMING_SNAKE_CASE      ,     **SCREAMING_SNAKE_CASE      ,     )

				SCREAMING_SNAKE_CASE_								=						json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
				if (
				    normalizer_state.get('lowercase'      ,     SCREAMING_SNAKE_CASE      ) != do_lower_case
				    or normalizer_state.get('strip_accents'      ,     SCREAMING_SNAKE_CASE      ) != strip_accents
				    or normalizer_state.get('handle_chinese_chars'      ,     SCREAMING_SNAKE_CASE      ) != tokenize_chinese_chars
				):
							SCREAMING_SNAKE_CASE_								=						getattr(SCREAMING_SNAKE_CASE      ,     normalizer_state.pop('type'      )      )
							SCREAMING_SNAKE_CASE_								=						do_lower_case
							SCREAMING_SNAKE_CASE_								=						strip_accents
							SCREAMING_SNAKE_CASE_								=						tokenize_chinese_chars
							SCREAMING_SNAKE_CASE_								=						normalizer_class(**SCREAMING_SNAKE_CASE      )

				SCREAMING_SNAKE_CASE_								=						do_lower_case





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     **SCREAMING_SNAKE_CASE      )  ->					Tuple:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						PaddingStrategy.MAX_LENGTH

				SCREAMING_SNAKE_CASE_								=						text
				SCREAMING_SNAKE_CASE_								=						kwargs.pop('text_pair'      ,     SCREAMING_SNAKE_CASE      )
				SCREAMING_SNAKE_CASE_								=						kwargs.pop('return_tensors'      ,     SCREAMING_SNAKE_CASE      )

				SCREAMING_SNAKE_CASE_								=						{
				    'input_ids': [],
				    'attention_mask': [],
				    'token_type_ids': [],
				}

				for idx, candidate_text in enumerate(SCREAMING_SNAKE_CASE      ):
							if batch_text_pair is not None:
										SCREAMING_SNAKE_CASE_								=						batch_text_pair[idx]
							else:
										SCREAMING_SNAKE_CASE_								=						None

							SCREAMING_SNAKE_CASE_								=						super().__call__(SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE      ,     return_tensors=SCREAMING_SNAKE_CASE      ,     **SCREAMING_SNAKE_CASE      )

							SCREAMING_SNAKE_CASE_								=						encoded_candidates.get('input_ids'      )
							SCREAMING_SNAKE_CASE_								=						encoded_candidates.get('attention_mask'      )
							SCREAMING_SNAKE_CASE_								=						encoded_candidates.get('token_type_ids'      )

							if encoded_input_ids is not None:
										output_data["input_ids"].append(SCREAMING_SNAKE_CASE      )
							if encoded_attention_mask is not None:
										output_data["attention_mask"].append(SCREAMING_SNAKE_CASE      )
							if encoded_token_type_ids is not None:
										output_data["token_type_ids"].append(SCREAMING_SNAKE_CASE      )

				SCREAMING_SNAKE_CASE_								=						{key: item for key, item in output_data.items() if len(SCREAMING_SNAKE_CASE      ) != 0}

				return BatchEncoding(SCREAMING_SNAKE_CASE      ,     tensor_type=SCREAMING_SNAKE_CASE      )





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE=None      )  ->					List[str]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE = None      )  ->					List[int]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						[self.sep_token_id]
				SCREAMING_SNAKE_CASE_								=						[self.cls_token_id]
				if token_ids_a is None:
							return len(cls + token_ids_a + sep      ) * [0]
				return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE = None      )  ->					Tuple[str]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						self._tokenizer.model.save(SCREAMING_SNAKE_CASE      ,     name=SCREAMING_SNAKE_CASE      )
				return tuple(SCREAMING_SNAKE_CASE      )






def lowercase    (  SCREAMING_SNAKE_CASE )			->							int:


			'''simple docstring'''







			if not numbers:
						return 0

			if not isinstance(SCREAMING_SNAKE_CASE       ,    (list, tuple) ) or not all(
			    isinstance(SCREAMING_SNAKE_CASE       ,    SCREAMING_SNAKE_CASE ) for number in numbers ):
						raise ValueError('numbers must be an iterable of integers' )

			SCREAMING_SNAKE_CASE_								=						SCREAMING_SNAKE_CASE_								=						SCREAMING_SNAKE_CASE_								=						numbers[0]

			for i in range(1       ,    len(SCREAMING_SNAKE_CASE ) ):
						# update the maximum and minimum subarray products
						SCREAMING_SNAKE_CASE_								=						numbers[i]
						if number < 0:
									SCREAMING_SNAKE_CASE_						,		SCREAMING_SNAKE_CASE_								=						min_till_now, max_till_now
						SCREAMING_SNAKE_CASE_								=						max(SCREAMING_SNAKE_CASE       ,    max_till_now * number )
						SCREAMING_SNAKE_CASE_								=						min(SCREAMING_SNAKE_CASE       ,    min_till_now * number )

						# update the maximum product found till now
						SCREAMING_SNAKE_CASE_								=						max(SCREAMING_SNAKE_CASE       ,    SCREAMING_SNAKE_CASE )

			return max_prod
