

'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


UpperCamelCase_    						=       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase_    						=       {
    """vocab_file""": {
        """unc-nlp/lxmert-base-uncased""": """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """unc-nlp/lxmert-base-uncased""": (
            """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCamelCase_    						=       {
    """unc-nlp/lxmert-base-uncased""": 512,
}

UpperCamelCase_    						=       {
    """unc-nlp/lxmert-base-uncased""": {"""do_lower_case""": True},
}





class 		__SCREAMING_SNAKE_CASE							(			lowercase__      ):
				lowerCamelCase_		       =    VOCAB_FILES_NAMES
				lowerCamelCase_		       =    PRETRAINED_VOCAB_FILES_MAP
				lowerCamelCase_		       =    PRETRAINED_INIT_CONFIGURATION
				lowerCamelCase_		       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCamelCase_		       =    LxmertTokenizer







				def __init__(       self     :		str		,					UpperCAmelCase__     :		str=None		,					UpperCAmelCase__     :		List[Any]=None		,					UpperCAmelCase__     :		List[Any]=True		,					UpperCAmelCase__     :		int="[UNK]"		,					UpperCAmelCase__     :		Tuple="[SEP]"		,					UpperCAmelCase__     :		str="[PAD]"		,					UpperCAmelCase__     :		Optional[Any]="[CLS]"		,					UpperCAmelCase__     :		Tuple="[MASK]"		,					UpperCAmelCase__     :		Tuple=True		,					UpperCAmelCase__     :		int=None		,					**UpperCAmelCase__     :		Optional[Any]		,					):
										'''simple docstring'''
										super().__init__(
										    UpperCAmelCase__		,					tokenizer_file=UpperCAmelCase__		,					do_lower_case=UpperCAmelCase__		,					unk_token=UpperCAmelCase__		,					sep_token=UpperCAmelCase__		,					pad_token=UpperCAmelCase__		,					cls_token=UpperCAmelCase__		,					mask_token=UpperCAmelCase__		,					tokenize_chinese_chars=UpperCAmelCase__		,					strip_accents=UpperCAmelCase__		,					**UpperCAmelCase__		,					)

										lowercase :							Union[str, Any]    		=json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
										if (
										    normalizer_state.get('''lowercase'''		,					UpperCAmelCase__   ) != do_lower_case
										    or normalizer_state.get('''strip_accents'''		,					UpperCAmelCase__   ) != strip_accents
										    or normalizer_state.get('''handle_chinese_chars'''		,					UpperCAmelCase__   ) != tokenize_chinese_chars
										):
																lowercase :							Union[str, Any]    		=getattr(UpperCAmelCase__		,					normalizer_state.pop('''type'''   )   )
																lowercase :							int    		=do_lower_case
																lowercase :							Tuple    		=strip_accents
																lowercase :							Any    		=tokenize_chinese_chars
																lowercase :							Union[str, Any]    		=normalizer_class(**UpperCAmelCase__   )

										lowercase :							int    		=do_lower_case







				def  lowerCamelCase_	(       self     :		List[Any]		,					UpperCAmelCase__     :		Any		,					UpperCAmelCase__     :		int=None   ):
										'''simple docstring'''
										lowercase :							Union[str, Any]    		=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

										return output







				def  lowerCamelCase_	(       self     :		Dict		,					UpperCAmelCase__     :		List[int]		,					UpperCAmelCase__     :		Optional[List[int]] = None   ):
										'''simple docstring'''
										lowercase :							List[Any]    		=[self.sep_token_id]
										lowercase :							List[str]    		=[self.cls_token_id]
										if token_ids_a is None:
																return len(cls + token_ids_a + sep   ) * [0]
										return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]







				def  lowerCamelCase_	(       self     :		str		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Optional[str] = None   ):
										'''simple docstring'''
										lowercase :							List[Any]    		=self._tokenizer.model.save(UpperCAmelCase__		,					name=UpperCAmelCase__   )
										return tuple(UpperCAmelCase__   )








'''simple docstring'''





import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
UpperCamelCase_    						=       object()

# For specifying empty leaf dict `{}`
UpperCamelCase_    						=       object()






def 				_lowerCAmelCase			(   __magic_name__						:     int   , __magic_name__						:     int						)				->  Optional[int]:
						lowercase :							Optional[Any]    		=tuple((re.compile(x + '''$'''						) for x in qs)						)
						for i in range(len(__magic_name__						) - len(__magic_name__						) + 1						):
												lowercase :							Union[str, Any]    		=[x.match(__magic_name__						) for x, y in zip(__magic_name__   , ks[i:]						)]
												if matches and all(__magic_name__						):
																		return True
						return False






def 				_lowerCAmelCase			(   __magic_name__						:     Dict						)				->  List[str]:
						def replace(__magic_name__						:     Optional[int]   , __magic_name__						:     Optional[Any]						):
												for rule, replacement in rules:
																		if _match(__magic_name__   , __magic_name__						):
																								return replacement
												return val

						return replace






def 				_lowerCAmelCase			(   )				->  int:
						return [
						    # embeddings
						    (("transformer", "wpe", "embedding"), P('''mp'''   , __magic_name__						)),
						    (("transformer", "wte", "embedding"), P('''mp'''   , __magic_name__						)),
						    # atention
						    (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(__magic_name__   , '''mp'''						)),
						    (("attention", "out_proj", "kernel"), P('''mp'''   , __magic_name__						)),
						    (("attention", "out_proj", "bias"), None),
						    # mlp
						    (("mlp", "c_fc", "kernel"), P(__magic_name__   , '''mp'''						)),
						    (("mlp", "c_fc", "bias"), P('''mp'''						)),
						    (("mlp", "c_proj", "kernel"), P('''mp'''   , __magic_name__						)),
						    (("mlp", "c_proj", "bias"), None),
						    # layer norms
						    ((r"ln_\d+", "bias"), None),
						    ((r"\d+", r"ln_\d+", "scale"), None),
						    (("ln_f", "bias"), None),
						    (("ln_f", "scale"), None),
						]







def 				_lowerCAmelCase			(   __magic_name__						:     str						)				->  int:
						lowercase :							int    		=_get_partition_rules()
						lowercase :							Tuple    		=_replacement_rules(__magic_name__						)
						lowercase :							Any    		={k: _unmatched for k in flatten_dict(__magic_name__						)}
						lowercase :							Any    		={k: replace(__magic_name__   , __magic_name__						) for k, v in initd.items()}
						assert _unmatched not in result.values(), "Incomplete partition spec."
						return freeze(unflatten_dict(__magic_name__						)						)





