

'''simple docstring'''





import contextlib
from multiprocessing import Pool, RLock

from tqdm.auto import tqdm

from ..utils import experimental, logging


UpperCamelCase_    						=       logging.get_logger(__name__)





class 		__SCREAMING_SNAKE_CASE							:
				lowerCamelCase_		       =    None






@experimental
def 				_lowerCAmelCase			(   __magic_name__						:     Union[str, Any]   , __magic_name__						:     Tuple   , __magic_name__						:     List[Any]   , __magic_name__						:     Any   , __magic_name__						:     List[str]   , __magic_name__						:     int   , __magic_name__						:     List[str]						)				->  Any:
						if ParallelBackendConfig.backend_name is None:
												return _map_with_multiprocessing_pool(
												    __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__						)

						return _map_with_joblib(__magic_name__   , __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__						)






def 				_lowerCAmelCase			(   __magic_name__						:     str   , __magic_name__						:     Optional[int]   , __magic_name__						:     List[str]   , __magic_name__						:     Any   , __magic_name__						:     List[Any]   , __magic_name__						:     Optional[Any]   , __magic_name__						:     Optional[Any]						)				->  int:
						lowercase :							Optional[int]    		=num_proc if num_proc <= len(__magic_name__						) else len(__magic_name__						)
						lowercase :							Any    		=[]  # We organize the splits ourselve (contiguous splits)
						for index in range(__magic_name__						):
												lowercase :							Tuple    		=len(__magic_name__						) // num_proc
												lowercase :							Any    		=len(__magic_name__						) % num_proc
												lowercase :							Tuple    		=div * index + min(__magic_name__   , __magic_name__						)
												lowercase :							Optional[Any]    		=start + div + (1 if index < mod else 0)
												split_kwds.append((function, iterable[start:end], types, index, disable_tqdm, desc)						)

						if len(__magic_name__						) != sum(len(i[1]						) for i in split_kwds						):
												raise ValueError(
												    f'''Error dividing inputs iterable among processes. '''
												    f'''Total number of objects {len(__magic_name__						)}, '''
												    f'''length: {sum(len(i[1]						) for i in split_kwds						)}'''						)

						logger.info(
						    f'''Spawning {num_proc} processes for {len(__magic_name__						)} objects in slices of {[len(i[1]						) for i in split_kwds]}'''						)
						lowercase		,	lowercase :							Union[str, Any]    		=None, None
						if not disable_tqdm:
												lowercase		,	lowercase :							Tuple    		=(RLock(),), tqdm.set_lock
						with Pool(__magic_name__   , initargs=__magic_name__   , initializer=__magic_name__						) as pool:
												lowercase :							Any    		=pool.map(__magic_name__   , __magic_name__						)
						logger.info(f'''Finished {num_proc} processes'''						)
						lowercase :							Dict    		=[obj for proc_res in mapped for obj in proc_res]
						logger.info(f'''Unpacked {len(__magic_name__						)} objects'''						)

						return mapped






def 				_lowerCAmelCase			(   __magic_name__						:     List[Any]   , __magic_name__						:     int   , __magic_name__						:     Tuple   , __magic_name__						:     Tuple   , __magic_name__						:     List[Any]   , __magic_name__						:     Any   , __magic_name__						:     Any						)				->  str:
						# progress bar is not yet supported for _map_with_joblib, because tqdm couldn't accurately be applied to joblib,
						# and it requires monkey-patching joblib internal classes which is subject to change
						import joblib

						with joblib.parallel_backend(ParallelBackendConfig.backend_name   , n_jobs=__magic_name__						):
												return joblib.Parallel()(
												    joblib.delayed(__magic_name__						)((function, obj, types, None, True, None)						) for obj in iterable						)







@experimental
@contextlib.contextmanager
def 				_lowerCAmelCase			(   __magic_name__						:     str						)				->  Union[str, Any]:
						lowercase :							int    		=backend_name

						if backend_name == "spark":
												from joblibspark import register_spark

												register_spark()

												# TODO: call create_cache_and_write_probe if "download" in steps
												# TODO: raise NotImplementedError when Dataset.map etc is called

						try:
												yield
						finally:
												lowercase :							Tuple    		=None













import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


UpperCAmelCase__							  =						logging.get_logger(__name__)

UpperCAmelCase__							  =						{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__							  =						{
    "vocab_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"
        ),
        "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt",
        "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-openqa": (
            "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-reader": (
            "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-openqa": (
            "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-reader": (
            "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"
        ),
    },
}

UpperCAmelCase__							  =						{
    "google/realm-cc-news-pretrained-embedder": 512,
    "google/realm-cc-news-pretrained-encoder": 512,
    "google/realm-cc-news-pretrained-scorer": 512,
    "google/realm-cc-news-pretrained-openqa": 512,
    "google/realm-orqa-nq-openqa": 512,
    "google/realm-orqa-nq-reader": 512,
    "google/realm-orqa-wq-openqa": 512,
    "google/realm-orqa-wq-reader": 512,
}

UpperCAmelCase__							  =						{
    "google/realm-cc-news-pretrained-embedder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-encoder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-scorer": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-reader": {"do_lower_case": True},
    "google/realm-orqa-wq-openqa": {"do_lower_case": True},
    "google/realm-orqa-wq-reader": {"do_lower_case": True},
}






class 						lowercase_   (      lowercase  ):






      '''simple docstring'''




      __snake_case	    =    VOCAB_FILES_NAMES
      __snake_case	    =    PRETRAINED_VOCAB_FILES_MAP
      __snake_case	    =    PRETRAINED_INIT_CONFIGURATION
      __snake_case	    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __snake_case	    =    RealmTokenizer







      def __init__( self :		List[Any]					,   __UpperCAmelCase :		Union[str, Any]=None					,   __UpperCAmelCase :		Tuple=None					,   __UpperCAmelCase :		List[Any]=True					,   __UpperCAmelCase :		Any="[UNK]"					,   __UpperCAmelCase :		Dict="[SEP]"					,   __UpperCAmelCase :		List[str]="[PAD]"					,   __UpperCAmelCase :		str="[CLS]"					,   __UpperCAmelCase :		Optional[int]="[MASK]"					,   __UpperCAmelCase :		str=True					,   __UpperCAmelCase :		List[str]=None					,   **__UpperCAmelCase :		str					,   )       ->Optional[int]:




       """simple docstring"""


       super().__init__(
           __UpperCAmelCase					,   tokenizer_file=__UpperCAmelCase					,   do_lower_case=__UpperCAmelCase					,   unk_token=__UpperCAmelCase					,   sep_token=__UpperCAmelCase					,   pad_token=__UpperCAmelCase					,   cls_token=__UpperCAmelCase					,   mask_token=__UpperCAmelCase					,   tokenize_chinese_chars=__UpperCAmelCase					,   strip_accents=__UpperCAmelCase					,   **__UpperCAmelCase					,   )

       a						    = json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
       if (
           normalizer_state.get('''lowercase'''					,   __UpperCAmelCase    ) != do_lower_case
           or normalizer_state.get('''strip_accents'''					,   __UpperCAmelCase    ) != strip_accents
           or normalizer_state.get('''handle_chinese_chars'''					,   __UpperCAmelCase    ) != tokenize_chinese_chars
       ):
        a						    = getattr(__UpperCAmelCase					,   normalizer_state.pop('''type'''    )    )
        a						    = do_lower_case
        a						    = strip_accents
        a						    = tokenize_chinese_chars
        a						    = normalizer_class(**__UpperCAmelCase    )

       a						    = do_lower_case







      def  __lowerCAmelCase   ( self :		str					,   __UpperCAmelCase :		List[Any]					,   **__UpperCAmelCase :		List[str]    )       ->List[Any]:




       """simple docstring"""


       a						    = PaddingStrategy.MAX_LENGTH

       a						    = text
       a						    = kwargs.pop('''text_pair'''					,   __UpperCAmelCase    )
       a						    = kwargs.pop('''return_tensors'''					,   __UpperCAmelCase    )

       a						    = {
           '''input_ids''': [],
           '''attention_mask''': [],
           '''token_type_ids''': [],
       }

       for idx, candidate_text in enumerate(__UpperCAmelCase    ):
        if batch_text_pair is not None:
         a						    = batch_text_pair[idx]
        else:
         a						    = None

        a						    = super().__call__(__UpperCAmelCase					,   __UpperCAmelCase					,   return_tensors=__UpperCAmelCase					,   **__UpperCAmelCase    )

        a						    = encoded_candidates.get('''input_ids'''    )
        a						    = encoded_candidates.get('''attention_mask'''    )
        a						    = encoded_candidates.get('''token_type_ids'''    )

        if encoded_input_ids is not None:
         output_data["input_ids"].append(__UpperCAmelCase    )
        if encoded_attention_mask is not None:
         output_data["attention_mask"].append(__UpperCAmelCase    )
        if encoded_token_type_ids is not None:
         output_data["token_type_ids"].append(__UpperCAmelCase    )

       a						    = {key: item for key, item in output_data.items() if len(__UpperCAmelCase    ) != 0}

       return BatchEncoding(__UpperCAmelCase					,   tensor_type=__UpperCAmelCase    )







      def  __lowerCAmelCase   ( self :		Tuple					,   __UpperCAmelCase :		Optional[int]					,   __UpperCAmelCase :		Dict=None    )       ->str:




       """simple docstring"""


       a						    = [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

       return output







      def  __lowerCAmelCase   ( self :		str					,   __UpperCAmelCase :		List[int]					,   __UpperCAmelCase :		Optional[List[int]] = None    )       ->List[int]:




       """simple docstring"""


       a						    = [self.sep_token_id]
       a						    = [self.cls_token_id]
       if token_ids_a is None:
        return len(cls + token_ids_a + sep    ) * [0]
       return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]







      def  __lowerCAmelCase   ( self :		Dict					,   __UpperCAmelCase :		str					,   __UpperCAmelCase :		Optional[str] = None    )       ->Tuple[str]:




       """simple docstring"""


       a						    = self._tokenizer.model.save(__UpperCAmelCase					,   name=__UpperCAmelCase    )
       return tuple(__UpperCAmelCase    )


