'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowerCAmelCase_	:		List[str]			=     logging.get_logger(__name__)

lowerCAmelCase_	:		Dict			=     {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase_	:		Optional[Any]			=     {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCAmelCase_	:		List[str]			=     {
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

lowerCAmelCase_	:		int			=     {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}




class 		SCREAMING_SNAKE_CASE     (					SCREAMING_SNAKE_CASE	):







				'''simple docstring'''
				UpperCAmelCase__							=							VOCAB_FILES_NAMES
				UpperCAmelCase__							=							PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase__							=							PRETRAINED_INIT_CONFIGURATION
				UpperCAmelCase__							=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase__							=							RealmTokenizer





				def __init__(						self					:						Optional[Any]       ,		lowercase__					:						Any=None       ,		lowercase__					:						Union[str, Any]=None       ,		lowercase__					:						Optional[int]=True       ,		lowercase__					:						Dict="[UNK]"       ,		lowercase__					:						List[Any]="[SEP]"       ,		lowercase__					:						Optional[int]="[PAD]"       ,		lowercase__					:						Optional[int]="[CLS]"       ,		lowercase__					:						List[Any]="[MASK]"       ,		lowercase__					:						str=True       ,		lowercase__					:						Union[str, Any]=None       ,		**lowercase__					:						Union[str, Any]       ,		)							->Dict:


										'''simple docstring'''




										super().__init__(
										    lowercase__       ,		tokenizer_file=lowercase__       ,		do_lower_case=lowercase__       ,		unk_token=lowercase__       ,		sep_token=lowercase__       ,		pad_token=lowercase__       ,		cls_token=lowercase__       ,		mask_token=lowercase__       ,		tokenize_chinese_chars=lowercase__       ,		strip_accents=lowercase__       ,		**lowercase__       ,		)

										_UpperCamelCase			:			Optional[int]   			=						json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
										if (
										    normalizer_state.get("lowercase"       ,		lowercase__					) != do_lower_case
										    or normalizer_state.get("strip_accents"       ,		lowercase__					) != strip_accents
										    or normalizer_state.get("handle_chinese_chars"       ,		lowercase__					) != tokenize_chinese_chars
										):
																_UpperCamelCase			:			Tuple   			=						getattr(lowercase__       ,		normalizer_state.pop("type"					)					)
																_UpperCamelCase			:			Dict   			=						do_lower_case
																_UpperCamelCase			:			List[str]   			=						strip_accents
																_UpperCamelCase			:			int   			=						tokenize_chinese_chars
																_UpperCamelCase			:			List[Any]   			=						normalizer_class(**lowercase__					)

										_UpperCamelCase			:			Optional[int]   			=						do_lower_case





				def  snake_case__				(						self					:						int       ,		lowercase__					:						Optional[Any]       ,		**lowercase__					:						Any					)							->Any:


										'''simple docstring'''




										_UpperCamelCase			:			List[Any]   			=						PaddingStrategy.MAX_LENGTH

										_UpperCamelCase			:			int   			=						text
										_UpperCamelCase			:			str   			=						kwargs.pop("text_pair"       ,		lowercase__					)
										_UpperCamelCase			:			Optional[Any]   			=						kwargs.pop("return_tensors"       ,		lowercase__					)

										_UpperCamelCase			:			str   			=						{
										    "input_ids": [],
										    "attention_mask": [],
										    "token_type_ids": [],
										}

										for idx, candidate_text in enumerate(lowercase__					):
																if batch_text_pair is not None:
																						_UpperCamelCase			:			int   			=						batch_text_pair[idx]
																else:
																						_UpperCamelCase			:			Optional[int]   			=						None

																_UpperCamelCase			:			Optional[Any]   			=						super().__call__(lowercase__       ,		lowercase__       ,		return_tensors=lowercase__       ,		**lowercase__					)

																_UpperCamelCase			:			List[Any]   			=						encoded_candidates.get("input_ids"					)
																_UpperCamelCase			:			List[Any]   			=						encoded_candidates.get("attention_mask"					)
																_UpperCamelCase			:			int   			=						encoded_candidates.get("token_type_ids"					)

																if encoded_input_ids is not None:
																						output_data["input_ids"].append(lowercase__					)
																if encoded_attention_mask is not None:
																						output_data["attention_mask"].append(lowercase__					)
																if encoded_token_type_ids is not None:
																						output_data["token_type_ids"].append(lowercase__					)

										_UpperCamelCase			:			int   			=						{key: item for key, item in output_data.items() if len(lowercase__					) != 0}

										return BatchEncoding(lowercase__       ,		tensor_type=lowercase__					)





				def  snake_case__				(						self					:						Union[str, Any]       ,		lowercase__					:						Union[str, Any]       ,		lowercase__					:						List[Any]=None					)							->List[str]:


										'''simple docstring'''




										_UpperCamelCase			:			Optional[Any]   			=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

										return output





				def  snake_case__				(						self					:						List[str]       ,		lowercase__					:						List[int]       ,		lowercase__					:						Optional[List[int]] = None					)							->List[int]:


										'''simple docstring'''




										_UpperCamelCase			:			Dict   			=						[self.sep_token_id]
										_UpperCamelCase			:			Any   			=						[self.cls_token_id]
										if token_ids_a is None:
																return len(cls + token_ids_a + sep					) * [0]
										return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]







				def  snake_case__				(						self					:						Any       ,		lowercase__					:						str       ,		lowercase__					:						Optional[str] = None					)							->Tuple[str]:


										'''simple docstring'''




										_UpperCamelCase			:			str   			=						self._tokenizer.model.save(lowercase__       ,		name=lowercase__					)
										return tuple(lowercase__					)



'''simple docstring'''





import time
import warnings
from abc import ABC
from copy import deepcopy
from typing import Optional

import torch

from ..utils import add_start_docstrings, logging


lowerCAmelCase_	:		Optional[int]			=     logging.get_logger(__name__)


lowerCAmelCase_	:		int			=     r"""
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        scores (`torch.FloatTensor` of shape `(batch_size, config.vocab_size)`):
            Prediction scores of a language modeling head. These can be scores for each vocabulary token before SoftMax
            or scores for each vocabulary token after SoftMax.
        kwargs (`Dict[str, Any]`, *optional*):
            Additional stopping criteria specific kwargs.

    Return:
        `bool`. `False` indicates we should continue, `True` indicates we should stop.

"""




class 		SCREAMING_SNAKE_CASE     (					SCREAMING_SNAKE_CASE	):







				'''simple docstring'''





				@add_start_docstrings(lowercase__					)
				def __call__(						self					:						str       ,		lowercase__					:						torch.LongTensor       ,		lowercase__					:						torch.FloatTensor       ,		**lowercase__					:						Tuple					)							->bool:


										'''simple docstring'''




										raise NotImplementedError("StoppingCriteria needs to be subclassed"					)





class 		SCREAMING_SNAKE_CASE     (					SCREAMING_SNAKE_CASE	):







				'''simple docstring'''





				def __init__(						self					:						Union[str, Any]       ,		lowercase__					:						int       ,		lowercase__					:						Optional[int] = None					)							->Union[str, Any]:


										'''simple docstring'''




										_UpperCamelCase			:			Optional[Any]   			=						max_length
										_UpperCamelCase			:			str   			=						max_position_embeddings







				@add_start_docstrings(lowercase__					)
				def __call__(						self					:						Optional[int]       ,		lowercase__					:						torch.LongTensor       ,		lowercase__					:						torch.FloatTensor       ,		**lowercase__					:						Optional[Any]					)							->bool:


										'''simple docstring'''




										_UpperCamelCase			:			List[Any]   			=						input_ids.shape[-1]
										_UpperCamelCase			:			Optional[int]   			=						cur_len >= self.max_length
										if self.max_position_embeddings is not None and not is_done and cur_len >= self.max_position_embeddings:
																logger.warning_once(
																    "This is a friendly reminder - the current text generation call will exceed the model's predefined "
																    f'''maximum length ({self.max_position_embeddings}). Depending on the model, you may observe '''
																    "exceptions, performance degradation, or nothing at all."					)
										return is_done





class 		SCREAMING_SNAKE_CASE     (					SCREAMING_SNAKE_CASE	):







				'''simple docstring'''





				def __init__(						self					:						List[Any]       ,		lowercase__					:						int       ,		lowercase__					:						int					)							->int:


										'''simple docstring'''




										warnings.warn(
										    "The class `MaxNewTokensCriteria` is deprecated. "
										    f'''Please use `MaxLengthCriteria(max_length={start_length + max_new_tokens})` '''
										    "with `max_length = start_length + max_new_tokens` instead."       ,		lowercase__       ,		)
										_UpperCamelCase			:			Dict   			=						start_length
										_UpperCamelCase			:			Optional[Any]   			=						max_new_tokens
										_UpperCamelCase			:			List[Any]   			=						start_length + max_new_tokens







				@add_start_docstrings(lowercase__					)
				def __call__(						self					:						Dict       ,		lowercase__					:						torch.LongTensor       ,		lowercase__					:						torch.FloatTensor       ,		**lowercase__					:						Dict					)							->bool:


										'''simple docstring'''




										return input_ids.shape[-1] >= self.max_length





class 		SCREAMING_SNAKE_CASE     (					SCREAMING_SNAKE_CASE	):







				'''simple docstring'''





				def __init__(						self					:						Union[str, Any]       ,		lowercase__					:						float       ,		lowercase__					:						Optional[float] = None					)							->Dict:


										'''simple docstring'''




										_UpperCamelCase			:			Dict   			=						max_time
										_UpperCamelCase			:			int   			=						time.time() if initial_timestamp is None else initial_timestamp







				@add_start_docstrings(lowercase__					)
				def __call__(						self					:						Dict       ,		lowercase__					:						torch.LongTensor       ,		lowercase__					:						torch.FloatTensor       ,		**lowercase__					:						Tuple					)							->bool:


										'''simple docstring'''




										return time.time() - self.initial_timestamp > self.max_time





class 		SCREAMING_SNAKE_CASE     (					SCREAMING_SNAKE_CASE	):







				'''simple docstring'''





				@add_start_docstrings(lowercase__					)
				def __call__(						self					:						Optional[int]       ,		lowercase__					:						torch.LongTensor       ,		lowercase__					:						torch.FloatTensor       ,		**lowercase__					:						List[Any]					)							->bool:


										'''simple docstring'''




										return any(criteria(lowercase__       ,		lowercase__					) for criteria in self					)







				@property
				def  snake_case__				(						self					:						Tuple					)							->Optional[int]:


										'''simple docstring'''




										for stopping_criterium in self:
																if isinstance(lowercase__       ,		lowercase__					):
																						return stopping_criterium.max_length
																elif isinstance(lowercase__       ,		lowercase__					):
																						return stopping_criterium.max_length
										return None





def 						__A							(     UpperCAmelCase     ,UpperCAmelCase					)     ->			StoppingCriteriaList:




						'''simple docstring'''
						_UpperCamelCase			:			Optional[int]   			=						stopping_criteria.max_length
						_UpperCamelCase			:			Optional[Any]   			=						deepcopy(UpperCAmelCase					)
						if stopping_max_length is not None and stopping_max_length != max_length:
												warnings.warn("You set different `max_length` for stopping criteria and `max_length` parameter"     ,UpperCAmelCase					)
						elif stopping_max_length is None:
												new_stopping_criteria.append(MaxLengthCriteria(max_length=UpperCAmelCase					)					)
						return new_stopping_criteria


