
'''simple docstring'''
from typing import List

import numpy as np







def 		A_(     A						:		dict):
							UpperCamelCase   			=     {key: len(A) for key, value in gen_kwargs.items() if isinstance(A		,     A)}
							if len(set(lists_lengths.values())) > 1:
														raise RuntimeError(
														    (
														        'Sharding is ambiguous for this dataset: '
														        + 'we found several data sources lists of different lengths, and we don\'t know over which list we should parallelize:\n'
														        + '\n'.join(f'''\t- key {key} has length {length}''' for key, length in lists_lengths.items())
														        + '\nTo fix this, check the \'gen_kwargs\' and make sure to use lists only for data sources, '
														        + 'and use tuples otherwise. In the end there should only be one single list, or several lists with the same length.'
														    ))
							UpperCamelCase   			=     max(lists_lengths.values()		,     default=0)
							return max(1		,     A)







def 		A_(     A						:		int		,     A						:		int):
							UpperCamelCase   			=     []
							for group_idx in range(A):
														UpperCamelCase   			=     num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs))
														if num_shards_to_add == 0:
																					break
														UpperCamelCase   			=     shards_indices_per_group[-1].stop if shards_indices_per_group else 0
														UpperCamelCase   			=     range(A		,     start + num_shards_to_add)
														shards_indices_per_group.append(A)
							return shards_indices_per_group







def 		A_(     A						:		dict		,     A						:		int):
							UpperCamelCase   			=     _number_of_shards_in_gen_kwargs(A)
							if num_shards == 1:
														return [dict(A)]
							else:
														UpperCamelCase   			=     _distribute_shards(num_shards=A		,     max_num_jobs=A)
														return [
														    {
														        key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]]
														        if isinstance(A		,     A)
														        else value
														        for key, value in gen_kwargs.items()
														    }
														    for group_idx in range(len(A))
														]







def 		A_(     A						:		List[dict]):
							return {
							    key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]]
							    if isinstance(gen_kwargs_list[0][key]		,     A)
							    else gen_kwargs_list[0][key]
							    for key in gen_kwargs_list[0]
							}







def 		A_(     A						:		np.random.Generator		,     A						:		dict):
							UpperCamelCase   			=     {len(A) for value in gen_kwargs.values() if isinstance(A		,     A)}
							UpperCamelCase   			=     {}
							for size in list_sizes:
														UpperCamelCase   			=     list(range(A))
														rng.shuffle(indices_per_size[size])
							# Now let's copy the gen_kwargs and shuffle the lists based on their sizes
							UpperCamelCase   			=     dict(A)
							for key, value in shuffled_kwargs.items():
														if isinstance(A		,     A):
																					UpperCamelCase   			=     [value[i] for i in indices_per_size[len(A)]]
							return shuffled_kwargs










'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


lowercase      :   Tuple      =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase      :   Dict      =       {
    """vocab_file""": {
        """unc-nlp/lxmert-base-uncased""": """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """unc-nlp/lxmert-base-uncased""": (
            """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase      :   List[Any]      =       {
    """unc-nlp/lxmert-base-uncased""": 512,
}

lowercase      :   Tuple      =       {
    """unc-nlp/lxmert-base-uncased""": {"""do_lower_case""": True},
}







class 					_a (a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Optional[int]          =					VOCAB_FILES_NAMES
     lowerCAmelCase_  :     Optional[int]          =					PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase_  :     Any          =					PRETRAINED_INIT_CONFIGURATION
     lowerCAmelCase_  :     Optional[int]          =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase_  :     Optional[int]          =					LxmertTokenizer







     def __init__(				self							,__a=None							,__a=None							,__a=True							,__a="[UNK]"							,__a="[SEP]"							,__a="[PAD]"							,__a="[CLS]"							,__a="[MASK]"							,__a=True							,__a=None							,**__a							,)      ->				str:
          super().__init__(
              __a							,tokenizer_file=__a							,do_lower_case=__a							,unk_token=__a							,sep_token=__a							,pad_token=__a							,cls_token=__a							,mask_token=__a							,tokenize_chinese_chars=__a							,strip_accents=__a							,**__a							,)

          snake_case			:      List[Any]        =			json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
          if (
              normalizer_state.get("""lowercase"""							,__a							) != do_lower_case
              or normalizer_state.get("""strip_accents"""							,__a							) != strip_accents
              or normalizer_state.get("""handle_chinese_chars"""							,__a							) != tokenize_chinese_chars
          ):
               snake_case			:      Union[str, Any]        =			getattr(__a							,normalizer_state.pop("""type"""							)							)
               snake_case			:      Optional[int]        =			do_lower_case
               snake_case			:      Optional[int]        =			strip_accents
               snake_case			:      List[Any]        =			tokenize_chinese_chars
               snake_case			:      Union[str, Any]        =			normalizer_class(**__a							)

          snake_case			:      str        =			do_lower_case







     def 			snake_case_      (				self							,__a							,__a=None							)      ->				Optional[int]:
          snake_case			:      Tuple        =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				List[int]:
          snake_case			:      int        =			[self.sep_token_id]
          snake_case			:      Any        =			[self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep							) * [0]
          return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				Tuple[str]:
          snake_case			:      Optional[int]        =			self._tokenizer.model.save(__a							,name=__a							)
          return tuple(__a							)



