
'''simple docstring'''
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


a_											=						{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

a_											=						{
    "vocab_file": {
        "unc-nlp/lxmert-base-uncased": "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "unc-nlp/lxmert-base-uncased": (
            "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"
        ),
    },
}

a_											=						{
    "unc-nlp/lxmert-base-uncased": 512,
}

a_											=						{
    "unc-nlp/lxmert-base-uncased": {"do_lower_case": True},
}






class    UpperCAmelCase_				(	snake_case__					):
 UpperCAmelCase_		    =						VOCAB_FILES_NAMES
 UpperCAmelCase_		    =						PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase_		    =						PRETRAINED_INIT_CONFIGURATION
 UpperCAmelCase_		    =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase_		    =						LxmertTokenizer





 def __init__(    self ,       lowercase_=None ,       lowercase_=None ,       lowercase_=True ,       lowercase_="[UNK]" ,       lowercase_="[SEP]" ,       lowercase_="[PAD]" ,       lowercase_="[CLS]" ,       lowercase_="[MASK]" ,       lowercase_=True ,       lowercase_=None ,       **lowercase_ ,       ):
     super().__init__(
         lowercase_ ,       tokenizer_file=lowercase_ ,       do_lower_case=lowercase_ ,       unk_token=lowercase_ ,       sep_token=lowercase_ ,       pad_token=lowercase_ ,       cls_token=lowercase_ ,       mask_token=lowercase_ ,       tokenize_chinese_chars=lowercase_ ,       strip_accents=lowercase_ ,       **lowercase_ ,       )

     snake_case_ :    Optional[int]     			=						json.loads(self.backend_tokenizer.normalizer.__getstate__())
     if (
         normalizer_state.get("lowercase" ,       lowercase_) != do_lower_case
         or normalizer_state.get("strip_accents" ,       lowercase_) != strip_accents
         or normalizer_state.get("handle_chinese_chars" ,       lowercase_) != tokenize_chinese_chars
     ):
         snake_case_ :    int     			=						getattr(lowercase_ ,       normalizer_state.pop("type"))
         snake_case_ :    Dict     			=						do_lower_case
         snake_case_ :    List[Any]     			=						strip_accents
         snake_case_ :    List[Any]     			=						tokenize_chinese_chars
         snake_case_ :    List[str]     			=						normalizer_class(**lowercase_)

     snake_case_ :    Optional[int]     			=						do_lower_case





 def        snake_case__		(    self ,       lowercase_ ,       lowercase_=None):
     snake_case_ :    Dict     			=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

     return output





 def        snake_case__		(    self ,       lowercase_ ,       lowercase_ = None):
     snake_case_ :    Optional[int]     			=						[self.sep_token_id]
     snake_case_ :    int     			=						[self.cls_token_id]
     if token_ids_a is None:
         return len(cls + token_ids_a + sep) * [0]
     return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





 def        snake_case__		(    self ,       lowercase_ ,       lowercase_ = None):
     snake_case_ :    List[str]     			=						self._tokenizer.model.save(lowercase_ ,       name=lowercase_)
     return tuple(lowercase_)






'''simple docstring'''
import contextlib
from multiprocessing import Pool, RLock

from tqdm.auto import tqdm

from ..utils import experimental, logging


a_											=						logging.get_logger(__name__)






class    UpperCAmelCase_				:
 UpperCAmelCase_		    =						None


@experimental
def 						UpperCamelCase_     (	__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       ):
    """simple docstring"""






    if ParallelBackendConfig.backend_name is None:
        return _map_with_multiprocessing_pool(
            __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       )

    return _map_with_joblib(__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       )

def 						UpperCamelCase_     (	__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       ):
    """simple docstring"""






    snake_case_ :    str     			=						num_proc if num_proc <= len(__SCREAMING_SNAKE_CASE       ) else len(__SCREAMING_SNAKE_CASE       )
    snake_case_ :    Optional[int]     			=						[]  # We organize the splits ourselve (contiguous splits)
    for index in range(__SCREAMING_SNAKE_CASE       ):
        snake_case_ :    List[str]     			=						len(__SCREAMING_SNAKE_CASE       ) // num_proc
        snake_case_ :    Optional[Any]     			=						len(__SCREAMING_SNAKE_CASE       ) % num_proc
        snake_case_ :    Optional[Any]     			=						div * index + min(__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       )
        snake_case_ :    Union[str, Any]     			=						start + div + (1 if index < mod else 0)
        split_kwds.append((function, iterable[start:end], types, index, disable_tqdm, desc)       )

    if len(__SCREAMING_SNAKE_CASE       ) != sum(len(i[1]       ) for i in split_kwds       ):
        raise ValueError(
            f'Error dividing inputs iterable among processes. '
            f'Total number of objects {len(__SCREAMING_SNAKE_CASE       )}, '
            f'length: {sum(len(i[1]       ) for i in split_kwds       )}'       )

    logger.info(
        f'Spawning {num_proc} processes for {len(__SCREAMING_SNAKE_CASE       )} objects in slices of {[len(i[1]       ) for i in split_kwds]}'       )
    snake_case_			, snake_case_ :    Tuple     			=						None, None
    if not disable_tqdm:
        snake_case_			, snake_case_ :    int     			=						(RLock(),), tqdm.set_lock
    with Pool(__SCREAMING_SNAKE_CASE,		initargs=__SCREAMING_SNAKE_CASE,		initializer=__SCREAMING_SNAKE_CASE       ) as pool:
        snake_case_ :    Union[str, Any]     			=						pool.map(__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       )
    logger.info(f'Finished {num_proc} processes'       )
    snake_case_ :    str     			=						[obj for proc_res in mapped for obj in proc_res]
    logger.info(f'Unpacked {len(__SCREAMING_SNAKE_CASE       )} objects'       )

    return mapped

def 						UpperCamelCase_     (	__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       ):
    """simple docstring"""






    import joblib

    with joblib.parallel_backend(ParallelBackendConfig.backend_name,		n_jobs=__SCREAMING_SNAKE_CASE       ):
        return joblib.Parallel()(
            joblib.delayed(__SCREAMING_SNAKE_CASE       )((function, obj, types, None, True, None)       ) for obj in iterable       )


@experimental
@contextlib.contextmanager
def 						UpperCamelCase_     (	__SCREAMING_SNAKE_CASE       ):
    """simple docstring"""






    snake_case_ :    Union[str, Any]     			=						backend_name

    if backend_name == "spark":
        from joblibspark import register_spark

        register_spark()

        # TODO: call create_cache_and_write_probe if "download" in steps
        # TODO: raise NotImplementedError when Dataset.map etc is called

    try:
        yield
    finally:
        snake_case_ :    Any     			=						None




