



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


__A       =			logging.get_logger(__name__)

__A       =			{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__A       =			{
    '''vocab_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'''
        ),
        '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt''',
        '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''google/realm-cc-news-pretrained-embedder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'''
        ),
        '''google/realm-cc-news-pretrained-encoder''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-scorer''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'''
        ),
        '''google/realm-cc-news-pretrained-openqa''': (
            '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-nq-reader''': (
            '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-openqa''': (
            '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'''
        ),
        '''google/realm-orqa-wq-reader''': (
            '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'''
        ),
    },
}

__A       =			{
    '''google/realm-cc-news-pretrained-embedder''': 512,
    '''google/realm-cc-news-pretrained-encoder''': 512,
    '''google/realm-cc-news-pretrained-scorer''': 512,
    '''google/realm-cc-news-pretrained-openqa''': 512,
    '''google/realm-orqa-nq-openqa''': 512,
    '''google/realm-orqa-nq-reader''': 512,
    '''google/realm-orqa-wq-openqa''': 512,
    '''google/realm-orqa-wq-reader''': 512,
}

__A       =			{
    '''google/realm-cc-news-pretrained-embedder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-encoder''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-scorer''': {'''do_lower_case''': True},
    '''google/realm-cc-news-pretrained-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-nq-reader''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-openqa''': {'''do_lower_case''': True},
    '''google/realm-orqa-wq-reader''': {'''do_lower_case''': True},
}






class 		lowercase						(			snake_case__):






	"""simple docstring"""


	a__				: int		       =							VOCAB_FILES_NAMES
	a__				: List[str]		       =							PRETRAINED_VOCAB_FILES_MAP
	a__				: List[str]		       =							PRETRAINED_INIT_CONFIGURATION
	a__				: Optional[Any]		       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a__				: Any		       =							RealmTokenizer




	def __init__(     self     :						List[str]						,  __UpperCAmelCase     :						List[Any]=None						,  __UpperCAmelCase     :						Optional[int]=None						,  __UpperCAmelCase     :						Any=True						,  __UpperCAmelCase     :						List[str]="[UNK]"						,  __UpperCAmelCase     :						Optional[int]="[SEP]"						,  __UpperCAmelCase     :						Any="[PAD]"						,  __UpperCAmelCase     :						List[str]="[CLS]"						,  __UpperCAmelCase     :						str="[MASK]"						,  __UpperCAmelCase     :						Dict=True						,  __UpperCAmelCase     :						Dict=None						,  **__UpperCAmelCase     :						Any						,  )      ->			Union[str, Any]:
							super().__init__(
							    __UpperCAmelCase						,  tokenizer_file=__UpperCAmelCase						,  do_lower_case=__UpperCAmelCase						,  unk_token=__UpperCAmelCase						,  sep_token=__UpperCAmelCase						,  pad_token=__UpperCAmelCase						,  cls_token=__UpperCAmelCase						,  mask_token=__UpperCAmelCase						,  tokenize_chinese_chars=__UpperCAmelCase						,  strip_accents=__UpperCAmelCase						,  **__UpperCAmelCase						,  )

							UpperCAmelCase_=       json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
							if (
							    normalizer_state.get("""lowercase"""						,  __UpperCAmelCase					) != do_lower_case
							    or normalizer_state.get("""strip_accents"""						,  __UpperCAmelCase					) != strip_accents
							    or normalizer_state.get("""handle_chinese_chars"""						,  __UpperCAmelCase					) != tokenize_chinese_chars
							):
													UpperCAmelCase_=       getattr(__UpperCAmelCase						,  normalizer_state.pop("""type"""					)					)
													UpperCAmelCase_=       do_lower_case
													UpperCAmelCase_=       strip_accents
													UpperCAmelCase_=       tokenize_chinese_chars
													UpperCAmelCase_=       normalizer_class(**__UpperCAmelCase					)

							UpperCAmelCase_=       do_lower_case




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[int]						,  __UpperCAmelCase     :						List[Any]						,  **__UpperCAmelCase     :						List[str]					)      ->			Optional[int]:
							UpperCAmelCase_=       PaddingStrategy.MAX_LENGTH

							UpperCAmelCase_=       text
							UpperCAmelCase_=       kwargs.pop("""text_pair"""						,  __UpperCAmelCase					)
							UpperCAmelCase_=       kwargs.pop("""return_tensors"""						,  __UpperCAmelCase					)

							UpperCAmelCase_=       {
							    """input_ids""": [],
							    """attention_mask""": [],
							    """token_type_ids""": [],
							}

							for idx, candidate_text in enumerate(__UpperCAmelCase					):
													if batch_text_pair is not None:
																			UpperCAmelCase_=       batch_text_pair[idx]
													else:
																			UpperCAmelCase_=       None

													UpperCAmelCase_=       super().__call__(__UpperCAmelCase						,  __UpperCAmelCase						,  return_tensors=__UpperCAmelCase						,  **__UpperCAmelCase					)

													UpperCAmelCase_=       encoded_candidates.get("""input_ids"""					)
													UpperCAmelCase_=       encoded_candidates.get("""attention_mask"""					)
													UpperCAmelCase_=       encoded_candidates.get("""token_type_ids"""					)

													if encoded_input_ids is not None:
																			output_data["input_ids"].append(__UpperCAmelCase					)
													if encoded_attention_mask is not None:
																			output_data["attention_mask"].append(__UpperCAmelCase					)
													if encoded_token_type_ids is not None:
																			output_data["token_type_ids"].append(__UpperCAmelCase					)

							UpperCAmelCase_=       {key: item for key, item in output_data.items() if len(__UpperCAmelCase					) != 0}

							return BatchEncoding(__UpperCAmelCase						,  tensor_type=__UpperCAmelCase					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[str]						,  __UpperCAmelCase     :						Any						,  __UpperCAmelCase     :						int=None					)      ->			int:
							UpperCAmelCase_=       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

							return output




	def   _SCREAMING_SNAKE_CASE		(     self     :						Union[str, Any]						,  __UpperCAmelCase     :						List[int]						,  __UpperCAmelCase     :						Optional[List[int]] = None					)      ->			List[int]:
							UpperCAmelCase_=       [self.sep_token_id]
							UpperCAmelCase_=       [self.cls_token_id]
							if token_ids_a is None:
													return len(cls + token_ids_a + sep					) * [0]
							return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]




	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict						,  __UpperCAmelCase     :						str						,  __UpperCAmelCase     :						Optional[str] = None					)      ->			Tuple[str]:
							UpperCAmelCase_=       self._tokenizer.model.save(__UpperCAmelCase						,  name=__UpperCAmelCase					)
							return tuple(__UpperCAmelCase					)









import warnings
from functools import wraps
from typing import Callable



def 	__a					(   lowerCAmelCase_       : Callable    )				->			Callable:






						'''simple docstring'''



						@wraps(lowerCAmelCase_    )
						def _inner_fn(*lowerCAmelCase_       : List[Any]      ,**lowerCAmelCase_       : Tuple    ):
												warnings.warn(
												    (F"""'{fn.__name__}' is experimental and might be subject to breaking changes in the future.""")      ,lowerCAmelCase_      ,)
												return fn(*lowerCAmelCase_      ,**lowerCAmelCase_    )

						return _inner_fn




