




import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
			from transformers.pipelines.conversational import Conversation


__SCREAMING_SNAKE_CASE 						=logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE 						={'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__SCREAMING_SNAKE_CASE 						={
    'tokenizer_file': {
        'EleutherAI/gpt-neox-20b': 'https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/tokenizer.json',
    },
}

__SCREAMING_SNAKE_CASE 						={
    'gpt-neox-20b': 2_048,
}






class  __magic_name__						(  SCREAMING_SNAKE_CASE_):
				'''simple docstring'''







				SCREAMING_SNAKE_CASE__					:							List[str]	      =					VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__					:							Any	      =					PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__					:							int	      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__					:							Optional[int]	      =					["input_ids", "attention_mask"]


				def __init__(							self:   Dict		,					_lowerCamelCase:   List[str]=None		,					_lowerCamelCase:   int=None		,					_lowerCamelCase:   int=None		,					_lowerCamelCase:   Optional[Any]="<|endoftext|>"		,					_lowerCamelCase:   Tuple="<|endoftext|>"		,					_lowerCamelCase:   Union[str, Any]="<|endoftext|>"		,					_lowerCamelCase:   Optional[Any]=False		,					**_lowerCamelCase:   Tuple		,					)     ->   List[Any]:
											super().__init__(
											    UpperCamelCase__		,					UpperCamelCase__		,					tokenizer_file=UpperCamelCase__		,					unk_token=UpperCamelCase__		,					bos_token=UpperCamelCase__		,					eos_token=UpperCamelCase__		,					add_prefix_space=UpperCamelCase__		,					**UpperCamelCase__		,					)

											SCREAMING_SNAKE_CASE_   	=    json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
											if pre_tok_state.get('''add_prefix_space'''		,					UpperCamelCase__		) != add_prefix_space:
																		SCREAMING_SNAKE_CASE_   	=    getattr(UpperCamelCase__		,					pre_tok_state.pop('''type'''		)		)
																		SCREAMING_SNAKE_CASE_   	=    add_prefix_space
																		SCREAMING_SNAKE_CASE_   	=    pre_tok_class(**UpperCamelCase__		)

											SCREAMING_SNAKE_CASE_   	=    add_prefix_space


				def     _A							(							self:   Union[str, Any]		,					_lowerCamelCase:   List[str]		,					_lowerCamelCase:   Any = None		)     ->   Union[str, Any]:
											SCREAMING_SNAKE_CASE_   	=    self._tokenizer.model.save(UpperCamelCase__		,					name=UpperCamelCase__		)
											return tuple(UpperCamelCase__		)





				def     _A							(							self:   Tuple		,					_lowerCamelCase:   Dict		)     ->   Optional[int]:
											SCREAMING_SNAKE_CASE_   	=    []
											for is_user, text in conversation.iter_texts():
																		input_ids.extend(self.encode(UpperCamelCase__		,					add_special_tokens=UpperCamelCase__		) + [self.eos_token_id]		)

											if len(UpperCamelCase__		) > self.model_max_length:
																		SCREAMING_SNAKE_CASE_   	=    input_ids[-self.model_max_length :]
											return input_ids








from ...configuration_utils import PretrainedConfig
from ...utils import logging


__SCREAMING_SNAKE_CASE 						=logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE 						={
    """google/realm-cc-news-pretrained-embedder""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-encoder""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-scorer""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/config.json"""
    ),
    """google/realm-cc-news-pretrained-openqa""": (
        """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/config.json"""
    ),
    """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/config.json""",
    """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/config.json""",
    """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/config.json""",
    """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/config.json""",
    # See all REALM models at https://huggingface.co/models?filter=realm
}






class  __magic_name__						(  __UpperCAmelCase):
				'''simple docstring'''







				SCREAMING_SNAKE_CASE__					:							int	      =					"realm"


				def __init__(							self:   Tuple		,					_lowerCamelCase:   Union[str, Any]=3_05_22		,					_lowerCamelCase:   Tuple=7_68		,					_lowerCamelCase:   str=1_28		,					_lowerCamelCase:   str=12		,					_lowerCamelCase:   int=12		,					_lowerCamelCase:   Union[str, Any]=8		,					_lowerCamelCase:   Optional[Any]=30_72		,					_lowerCamelCase:   str="gelu_new"		,					_lowerCamelCase:   str=0.1		,					_lowerCamelCase:   Union[str, Any]=0.1		,					_lowerCamelCase:   Optional[int]=5_12		,					_lowerCamelCase:   Union[str, Any]=2		,					_lowerCamelCase:   int=0.02		,					_lowerCamelCase:   Tuple=1E-12		,					_lowerCamelCase:   List[Any]=2_56		,					_lowerCamelCase:   Any=10		,					_lowerCamelCase:   Optional[Any]=1E-3		,					_lowerCamelCase:   Any=5		,					_lowerCamelCase:   List[str]=3_20		,					_lowerCamelCase:   List[str]=13_35_37_18		,					_lowerCamelCase:   str=50_00		,					_lowerCamelCase:   str=1		,					_lowerCamelCase:   str=0		,					_lowerCamelCase:   Dict=2		,					**_lowerCamelCase:   Tuple		,					):
											super().__init__(pad_token_id=_lowerCamelCase		,					bos_token_id=_lowerCamelCase		,					eos_token_id=_lowerCamelCase		,					**_lowerCamelCase		)

											# Common config
											SCREAMING_SNAKE_CASE_   	=    vocab_size
											SCREAMING_SNAKE_CASE_   	=    max_position_embeddings
											SCREAMING_SNAKE_CASE_   	=    hidden_size
											SCREAMING_SNAKE_CASE_   	=    retriever_proj_size
											SCREAMING_SNAKE_CASE_   	=    num_hidden_layers
											SCREAMING_SNAKE_CASE_   	=    num_attention_heads
											SCREAMING_SNAKE_CASE_   	=    num_candidates
											SCREAMING_SNAKE_CASE_   	=    intermediate_size
											SCREAMING_SNAKE_CASE_   	=    hidden_act
											SCREAMING_SNAKE_CASE_   	=    hidden_dropout_prob
											SCREAMING_SNAKE_CASE_   	=    attention_probs_dropout_prob
											SCREAMING_SNAKE_CASE_   	=    initializer_range
											SCREAMING_SNAKE_CASE_   	=    type_vocab_size
											SCREAMING_SNAKE_CASE_   	=    layer_norm_eps

											# Reader config
											SCREAMING_SNAKE_CASE_   	=    span_hidden_size
											SCREAMING_SNAKE_CASE_   	=    max_span_width
											SCREAMING_SNAKE_CASE_   	=    reader_layer_norm_eps
											SCREAMING_SNAKE_CASE_   	=    reader_beam_size
											SCREAMING_SNAKE_CASE_   	=    reader_seq_len

											# Retrieval config
											SCREAMING_SNAKE_CASE_   	=    num_block_records
											SCREAMING_SNAKE_CASE_   	=    searcher_beam_size


