






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


UpperCAmelCase__      			=						{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase__      			=						{
    '''vocab_file''': {
        '''unc-nlp/lxmert-base-uncased''': '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''unc-nlp/lxmert-base-uncased''': (
            '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCAmelCase__      			=						{
    '''unc-nlp/lxmert-base-uncased''': 512,
}

UpperCAmelCase__      			=						{
    '''unc-nlp/lxmert-base-uncased''': {'''do_lower_case''': True},
}




class 							snake_case_       (    __UpperCamelCase      ):

						"""simple docstring"""



						snake_case__        =      VOCAB_FILES_NAMES
						snake_case__        =      PRETRAINED_VOCAB_FILES_MAP
						snake_case__        =      PRETRAINED_INIT_CONFIGURATION
						snake_case__        =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						snake_case__        =      LxmertTokenizer



						def __init__(self:					str ,					__UpperCAmelCase:					Optional[int]=None ,					__UpperCAmelCase:					Union[str, Any]=None ,					__UpperCAmelCase:					str=True ,					__UpperCAmelCase:					int="[UNK]" ,					__UpperCAmelCase:					int="[SEP]" ,					__UpperCAmelCase:					Dict="[PAD]" ,					__UpperCAmelCase:					Tuple="[CLS]" ,					__UpperCAmelCase:					Dict="[MASK]" ,					__UpperCAmelCase:					Dict=True ,					__UpperCAmelCase:					Optional[Any]=None ,					**__UpperCAmelCase:					Any ,					)    ->     str:




											'''simple docstring'''



											super().__init__(
											    __UpperCAmelCase ,					tokenizer_file=__UpperCAmelCase ,					do_lower_case=__UpperCAmelCase ,					unk_token=__UpperCAmelCase ,					sep_token=__UpperCAmelCase ,					pad_token=__UpperCAmelCase ,					cls_token=__UpperCAmelCase ,					mask_token=__UpperCAmelCase ,					tokenize_chinese_chars=__UpperCAmelCase ,					strip_accents=__UpperCAmelCase ,					**__UpperCAmelCase ,					)

											__a					:						str      =					json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
											if (
											    normalizer_state.get("lowercase" ,					__UpperCAmelCase   ) != do_lower_case
											    or normalizer_state.get("strip_accents" ,					__UpperCAmelCase   ) != strip_accents
											    or normalizer_state.get("handle_chinese_chars" ,					__UpperCAmelCase   ) != tokenize_chinese_chars
											):
																__a					:						Optional[int]      =					getattr(__UpperCAmelCase ,					normalizer_state.pop("type"   )   )
																__a					:						int      =					do_lower_case
																__a					:						str      =					strip_accents
																__a					:						Tuple      =					tokenize_chinese_chars
																__a					:						List[str]      =					normalizer_class(**__UpperCAmelCase   )

											__a					:						Dict      =					do_lower_case



						def        UpperCAmelCase__					(self:					int ,					__UpperCAmelCase:					List[Any] ,					__UpperCAmelCase:					Tuple=None   )    ->     Union[str, Any]:




											'''simple docstring'''



											__a					:						int      =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

											return output



						def        UpperCAmelCase__					(self:					Tuple ,					__UpperCAmelCase:					List[int] ,					__UpperCAmelCase:					Optional[List[int]] = None   )    ->     List[int]:




											'''simple docstring'''



											__a					:						Optional[int]      =					[self.sep_token_id]
											__a					:						List[str]      =					[self.cls_token_id]
											if token_ids_a is None:
																return len(cls + token_ids_a + sep   ) * [0]
											return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



						def        UpperCAmelCase__					(self:					int ,					__UpperCAmelCase:					str ,					__UpperCAmelCase:					Optional[str] = None   )    ->     Tuple[str]:




											'''simple docstring'''



											__a					:						Any      =					self._tokenizer.model.save(__UpperCAmelCase ,					name=__UpperCAmelCase   )
											return tuple(__UpperCAmelCase   )













import unittest

from transformers import is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, require_torch, slow


if is_flax_available():
			import optax
			from flax.training.common_utils import onehot

			from transformers import AutoTokenizer, FlaxMTaForConditionalGeneration
			from transformers.models.ta.modeling_flax_ta import shift_tokens_right




@require_torch
@require_sentencepiece
@require_tokenizers
@require_flax
class 							snake_case_       (    unittest.TestCase      ):

						"""simple docstring"""



						@slow
						def        UpperCAmelCase__					(self:					Optional[int]   )    ->     List[str]:




											'''simple docstring'''



											__a					:						Optional[Any]      =					FlaxMTaForConditionalGeneration.from_pretrained("google/mt5-small"   )
											__a					:						Dict      =					AutoTokenizer.from_pretrained("google/mt5-small"   )

											__a					:						Any      =					tokenizer("Hello there" ,					return_tensors="np"   ).input_ids
											__a					:						Union[str, Any]      =					tokenizer("Hi I am" ,					return_tensors="np"   ).input_ids

											__a					:						Union[str, Any]      =					shift_tokens_right(__UpperCAmelCase ,					model.config.pad_token_id ,					model.config.decoder_start_token_id   )

											__a					:						Tuple      =					model(__UpperCAmelCase ,					decoder_input_ids=__UpperCAmelCase   ).logits
											__a					:						str      =					optax.softmax_cross_entropy(__UpperCAmelCase ,					onehot(__UpperCAmelCase ,					logits.shape[-1]   )   ).mean()

											__a					:						List[str]      =					-(labels.shape[-1] * loss.item())

											__a					:						Optional[int]      =					-84.91_27
											self.assertTrue(abs(mtf_score - EXPECTED_SCORE   ) < 1E-4   )





