





from typing import List

import numpy as np



def      _lowerCAmelCase		(	_lowerCAmelCase  ):
   '''simple docstring'''


   A_  :					List[Any]         =						{key: len(__snake_case  ) for key, value in gen_kwargs.items() if isinstance(__snake_case	,__snake_case  )}
   if len(set(lists_lengths.values()  )  ) > 1:
      raise RuntimeError(
          (
              """Sharding is ambiguous for this dataset: """
              + """we found several data sources lists of different lengths, and we don\'t know over which list we should parallelize:\n"""
              + """\n""".join(f"""\t- key {key} has length {length}""" for key, length in lists_lengths.items()  )
              + """\nTo fix this, check the \'gen_kwargs\' and make sure to use lists only for data sources, """
              + """and use tuples otherwise. In the end there should only be one single list, or several lists with the same length."""
          )  )
   A_  :					Union[str, Any]         =						max(lists_lengths.values()	,default=0  )
   return max(1	,__snake_case  )



def      _lowerCAmelCase		(	_lowerCAmelCase	,_lowerCAmelCase  ):
   '''simple docstring'''


   A_  :					Optional[Any]         =						[]
   for group_idx in range(__snake_case  ):
      A_  :					List[str]         =						num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs))
      if num_shards_to_add == 0:
         break
      A_  :					List[Any]         =						shards_indices_per_group[-1].stop if shards_indices_per_group else 0
      A_  :					Tuple         =						range(__snake_case	,start + num_shards_to_add  )
      shards_indices_per_group.append(__snake_case  )
   return shards_indices_per_group



def      _lowerCAmelCase		(	_lowerCAmelCase	,_lowerCAmelCase  ):
   '''simple docstring'''


   A_  :					Optional[int]         =						_number_of_shards_in_gen_kwargs(__snake_case  )
   if num_shards == 1:
      return [dict(__snake_case  )]
   else:
      A_  :					Any         =						_distribute_shards(num_shards=__snake_case	,max_num_jobs=__snake_case  )
      return [
          {
              key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]]
              if isinstance(__snake_case	,__snake_case  )
              else value
              for key, value in gen_kwargs.items()
          }
          for group_idx in range(len(__snake_case  )  )
      ]



def      _lowerCAmelCase		(	_lowerCAmelCase  ):
   '''simple docstring'''


   return {
       key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]]
       if isinstance(gen_kwargs_list[0][key]	,__snake_case  )
       else gen_kwargs_list[0][key]
       for key in gen_kwargs_list[0]
   }







def      _lowerCAmelCase		(	_lowerCAmelCase	,_lowerCAmelCase  ):
   '''simple docstring'''


   A_  :					Tuple         =						{len(__snake_case  ) for value in gen_kwargs.values() if isinstance(__snake_case	,__snake_case  )}
   A_  :					Dict         =						{}
   for size in list_sizes:
      A_  :					List[Any]         =						list(range(__snake_case  )  )
      rng.shuffle(indices_per_size[size]  )
   # Now let's copy the gen_kwargs and shuffle the lists based on their sizes
   A_  :					Dict         =						dict(__snake_case  )
   for key, value in shuffled_kwargs.items():
      if isinstance(__snake_case	,__snake_case  ):
         A_  :					Union[str, Any]         =						[value[i] for i in indices_per_size[len(__snake_case  )]]
   return shuffled_kwargs














"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


snake_case : List[Any]     		=		{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

snake_case : List[Any]     		=		{
    """vocab_file""": {
        """unc-nlp/lxmert-base-uncased""": """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """unc-nlp/lxmert-base-uncased""": (
            """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

snake_case : Optional[int]     		=		{
    """unc-nlp/lxmert-base-uncased""": 5_1_2,
}

snake_case : List[Any]     		=		{
    """unc-nlp/lxmert-base-uncased""": {"""do_lower_case""": True},
}



class        UpperCamelCase__ (				a_):







    """simple docstring"""
    __UpperCAmelCase           = VOCAB_FILES_NAMES
    __UpperCAmelCase           = PRETRAINED_VOCAB_FILES_MAP
    __UpperCAmelCase           = PRETRAINED_INIT_CONFIGURATION
    __UpperCAmelCase           = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __UpperCAmelCase           = LxmertTokenizer







    def __init__(		self :	Union[str, Any]				,    UpperCamelCase_ :	List[str]=None				,    UpperCamelCase_ :	Dict=None				,    UpperCamelCase_ :	Optional[int]=True				,    UpperCamelCase_ :	Dict="[UNK]"				,    UpperCamelCase_ :	str="[SEP]"				,    UpperCamelCase_ :	Dict="[PAD]"				,    UpperCamelCase_ :	Optional[int]="[CLS]"				,    UpperCamelCase_ :	Any="[MASK]"				,    UpperCamelCase_ :	Any=True				,    UpperCamelCase_ :	List[Any]=None				,    **UpperCamelCase_ :	Dict				,    ):





           '''simple docstring'''

           super().__init__(
               UpperCamelCase_				,    tokenizer_file=UpperCamelCase_				,    do_lower_case=UpperCamelCase_				,    unk_token=UpperCamelCase_				,    sep_token=UpperCamelCase_				,    pad_token=UpperCamelCase_				,    cls_token=UpperCamelCase_				,    mask_token=UpperCamelCase_				,    tokenize_chinese_chars=UpperCamelCase_				,    strip_accents=UpperCamelCase_				,    **UpperCamelCase_				,    )

           __magic_name__      				=    json.loads(self.backend_tokenizer.normalizer.__getstate__() )
           if (
               normalizer_state.get('lowercase'				,    UpperCamelCase_ ) != do_lower_case
               or normalizer_state.get('strip_accents'				,    UpperCamelCase_ ) != strip_accents
               or normalizer_state.get('handle_chinese_chars'				,    UpperCamelCase_ ) != tokenize_chinese_chars
           ):
                  __magic_name__      				=    getattr(UpperCamelCase_				,    normalizer_state.pop('type' ) )
                  __magic_name__      				=    do_lower_case
                  __magic_name__      				=    strip_accents
                  __magic_name__      				=    tokenize_chinese_chars
                  __magic_name__      				=    normalizer_class(**UpperCamelCase_ )

           __magic_name__      				=    do_lower_case







    def 				a__			(		self :	Dict				,    UpperCamelCase_ :	Optional[Any]				,    UpperCamelCase_ :	Optional[int]=None ):





           '''simple docstring'''

           __magic_name__      				=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output







    def 				a__			(		self :	int				,    UpperCamelCase_ :	List[int]				,    UpperCamelCase_ :	Optional[List[int]] = None ):





           '''simple docstring'''

           __magic_name__      				=    [self.sep_token_id]
           __magic_name__      				=    [self.cls_token_id]
           if token_ids_a is None:
                  return len(cls + token_ids_a + sep ) * [0]
           return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







    def 				a__			(		self :	Dict				,    UpperCamelCase_ :	str				,    UpperCamelCase_ :	Optional[str] = None ):





           '''simple docstring'''

           __magic_name__      				=    self._tokenizer.model.save(UpperCamelCase_				,    name=UpperCamelCase_ )
           return tuple(UpperCamelCase_ )