





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowerCAmelCase							:       int					 =	logging.get_logger(__name__)

lowerCAmelCase							:       str					 =	{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase							:       Dict					 =	{
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCAmelCase							:       Optional[int]					 =	{
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

lowerCAmelCase							:       str					 =	{
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}





class 				a			(      __lowercase			):
				SCREAMING_SNAKE_CASE__     :   List[str]		      =     VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__     :   Tuple		      =     PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__     :   Dict		      =     PRETRAINED_INIT_CONFIGURATION
				SCREAMING_SNAKE_CASE__     :   int		      =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__     :   List[str]		      =     RealmTokenizer






				def __init__(  self			, _lowerCAmelCase=None			, _lowerCAmelCase=None			, _lowerCAmelCase=True			, _lowerCAmelCase="[UNK]"			, _lowerCAmelCase="[SEP]"			, _lowerCAmelCase="[PAD]"			, _lowerCAmelCase="[CLS]"			, _lowerCAmelCase="[MASK]"			, _lowerCAmelCase=True			, _lowerCAmelCase=None			, **_lowerCAmelCase			, ):

										"""simple docstring"""




										super().__init__(
										    _lowerCAmelCase			, tokenizer_file=_lowerCAmelCase			, do_lower_case=_lowerCAmelCase			, unk_token=_lowerCAmelCase			, sep_token=_lowerCAmelCase			, pad_token=_lowerCAmelCase			, cls_token=_lowerCAmelCase			, mask_token=_lowerCAmelCase			, tokenize_chinese_chars=_lowerCAmelCase			, strip_accents=_lowerCAmelCase			, **_lowerCAmelCase			, )

										__SCREAMING_SNAKE_CASE:				List[Any]           =   json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
										if (
										    normalizer_state.get('''lowercase'''			, _lowerCAmelCase					) != do_lower_case
										    or normalizer_state.get('''strip_accents'''			, _lowerCAmelCase					) != strip_accents
										    or normalizer_state.get('''handle_chinese_chars'''			, _lowerCAmelCase					) != tokenize_chinese_chars
										):
																__SCREAMING_SNAKE_CASE:				List[Any]           =   getattr(_lowerCAmelCase			, normalizer_state.pop('''type'''					)					)
																__SCREAMING_SNAKE_CASE:				List[Any]           =   do_lower_case
																__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   strip_accents
																__SCREAMING_SNAKE_CASE:				List[str]           =   tokenize_chinese_chars
																__SCREAMING_SNAKE_CASE:				List[str]           =   normalizer_class(**_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				List[Any]           =   do_lower_case






				def 			snake_case_    (  self			, _lowerCAmelCase			, **_lowerCAmelCase					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[Any]           =   PaddingStrategy.MAX_LENGTH

										__SCREAMING_SNAKE_CASE:				Dict           =   text
										__SCREAMING_SNAKE_CASE:				List[Any]           =   kwargs.pop('''text_pair'''			, _lowerCAmelCase					)
										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   kwargs.pop('''return_tensors'''			, _lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   {
										    '''input_ids''': [],
										    '''attention_mask''': [],
										    '''token_type_ids''': [],
										}

										for idx, candidate_text in enumerate(_lowerCAmelCase					):
																if batch_text_pair is not None:
																						__SCREAMING_SNAKE_CASE:				List[Any]           =   batch_text_pair[idx]
																else:
																						__SCREAMING_SNAKE_CASE:				int           =   None

																__SCREAMING_SNAKE_CASE:				Any           =   super().__call__(_lowerCAmelCase			, _lowerCAmelCase			, return_tensors=_lowerCAmelCase			, **_lowerCAmelCase					)

																__SCREAMING_SNAKE_CASE:				List[str]           =   encoded_candidates.get('''input_ids'''					)
																__SCREAMING_SNAKE_CASE:				Tuple           =   encoded_candidates.get('''attention_mask'''					)
																__SCREAMING_SNAKE_CASE:				Optional[Any]           =   encoded_candidates.get('''token_type_ids'''					)

																if encoded_input_ids is not None:
																						output_data["input_ids"].append(_lowerCAmelCase					)
																if encoded_attention_mask is not None:
																						output_data["attention_mask"].append(_lowerCAmelCase					)
																if encoded_token_type_ids is not None:
																						output_data["token_type_ids"].append(_lowerCAmelCase					)

										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   {key: item for key, item in output_data.items() if len(_lowerCAmelCase					) != 0}

										return BatchEncoding(_lowerCAmelCase			, tensor_type=_lowerCAmelCase					)






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase=None					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				List[str]           =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

										return output






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase = None					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				str           =   [self.sep_token_id]
										__SCREAMING_SNAKE_CASE:				Optional[int]           =   [self.cls_token_id]
										if token_ids_a is None:
																return len(cls + token_ids_a + sep					) * [0]
										return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]






				def 			snake_case_    (  self			, _lowerCAmelCase			, _lowerCAmelCase = None					):

										"""simple docstring"""




										__SCREAMING_SNAKE_CASE:				str           =   self._tokenizer.model.save(_lowerCAmelCase			, name=_lowerCAmelCase					)
										return tuple(_lowerCAmelCase					)










from typing import Callable, Optional

from .. import Features
from ..packaged_modules.generator.generator import Generator
from .abc import AbstractDatasetInputStream





class 				a			(      __lowercase			):






				def __init__(  self			, _lowerCAmelCase			, _lowerCAmelCase = None			, _lowerCAmelCase = None			, _lowerCAmelCase = False			, _lowerCAmelCase = False			, _lowerCAmelCase = None			, _lowerCAmelCase = None			, **_lowerCAmelCase			, ):

										"""simple docstring"""




										super().__init__(
										    features=_lowerCAmelCase			, cache_dir=_lowerCAmelCase			, keep_in_memory=_lowerCAmelCase			, streaming=_lowerCAmelCase			, num_proc=_lowerCAmelCase			, **_lowerCAmelCase			, )
										__SCREAMING_SNAKE_CASE:				Any           =   Generator(
										    cache_dir=_lowerCAmelCase			, features=_lowerCAmelCase			, generator=_lowerCAmelCase			, gen_kwargs=_lowerCAmelCase			, **_lowerCAmelCase			, )






				def 			snake_case_    (  self					):

										"""simple docstring"""




										if self.streaming:
																__SCREAMING_SNAKE_CASE:				List[str]           =   self.builder.as_streaming_dataset(split='''train'''					)
										# Build regular (map-style) dataset
										else:
																__SCREAMING_SNAKE_CASE:				str           =   None
																__SCREAMING_SNAKE_CASE:				List[Any]           =   None
																__SCREAMING_SNAKE_CASE:				Tuple           =   None
																__SCREAMING_SNAKE_CASE:				Optional[Any]           =   None

																self.builder.download_and_prepare(
																    download_config=_lowerCAmelCase			, download_mode=_lowerCAmelCase			, verification_mode=_lowerCAmelCase			, base_path=_lowerCAmelCase			, num_proc=self.num_proc			, )
																__SCREAMING_SNAKE_CASE:				List[str]           =   self.builder.as_dataset(
																    split='''train'''			, verification_mode=_lowerCAmelCase			, in_memory=self.keep_in_memory					)
										return dataset



