


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


a_ :Dict  							=			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

a_ :Union[str, Any]  							=			{
    'vocab_file': {
        'unc-nlp/lxmert-base-uncased': 'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'unc-nlp/lxmert-base-uncased': (
            'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json'
        ),
    },
}

a_ :Tuple  							=			{
    'unc-nlp/lxmert-base-uncased': 5_12,
}

a_ :Dict  							=			{
    'unc-nlp/lxmert-base-uncased': {'do_lower_case': True},
}

class 							lowercase  (						__UpperCamelCase			):
     lowerCamelCase			:			Any      = VOCAB_FILES_NAMES
     lowerCamelCase			:			Dict      = PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase			:			str      = PRETRAINED_INIT_CONFIGURATION
     lowerCamelCase			:			Optional[Any]      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase			:			List[str]      = LxmertTokenizer







     def __init__(						self :						str   ,   _lowercase :						Optional[int]=None   ,   _lowercase :						Union[str, Any]=None   ,   _lowercase :						str=True   ,   _lowercase :						int="[UNK]"   ,   _lowercase :						int="[SEP]"   ,   _lowercase :						Dict="[PAD]"   ,   _lowercase :						Tuple="[CLS]"   ,   _lowercase :						Dict="[MASK]"   ,   _lowercase :						Dict=True   ,   _lowercase :						Optional[Any]=None   ,   **_lowercase :						Any   ,   ):
           super().__init__(
               __UpperCAmelCase   ,   tokenizer_file=__UpperCAmelCase   ,   do_lower_case=__UpperCAmelCase   ,   unk_token=__UpperCAmelCase   ,   sep_token=__UpperCAmelCase   ,   pad_token=__UpperCAmelCase   ,   cls_token=__UpperCAmelCase   ,   mask_token=__UpperCAmelCase   ,   tokenize_chinese_chars=__UpperCAmelCase   ,   strip_accents=__UpperCAmelCase   ,   **__UpperCAmelCase   ,   )

           SCREAMING_SNAKE_CASE__       :  str						     =		json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
           if (
               normalizer_state.get('''lowercase'''   ,   __UpperCAmelCase      ) != do_lower_case
               or normalizer_state.get('''strip_accents'''   ,   __UpperCAmelCase      ) != strip_accents
               or normalizer_state.get('''handle_chinese_chars'''   ,   __UpperCAmelCase      ) != tokenize_chinese_chars
           ):
                 SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		getattr(__UpperCAmelCase   ,   normalizer_state.pop('''type'''      )      )
                 SCREAMING_SNAKE_CASE__       :  int						     =		do_lower_case
                 SCREAMING_SNAKE_CASE__       :  str						     =		strip_accents
                 SCREAMING_SNAKE_CASE__       :  Tuple						     =		tokenize_chinese_chars
                 SCREAMING_SNAKE_CASE__       :  List[str]						     =		normalizer_class(**__UpperCAmelCase      )

           SCREAMING_SNAKE_CASE__       :  Dict						     =		do_lower_case







     def 							lowercase__       (						self :						int   ,   _lowercase :						List[Any]   ,   _lowercase :						Tuple=None      ):
           SCREAMING_SNAKE_CASE__       :  int						     =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                 output += token_ids_a + [self.sep_token_id]

           return output







     def 							lowercase__       (						self :						Tuple   ,   _lowercase :						List[int]   ,   _lowercase :						Optional[List[int]] = None      ):
           SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		[self.sep_token_id]
           SCREAMING_SNAKE_CASE__       :  List[str]						     =		[self.cls_token_id]
           if token_ids_a is None:
                 return len(cls + token_ids_a + sep      ) * [0]
           return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]







     def 							lowercase__       (						self :						int   ,   _lowercase :						str   ,   _lowercase :						Optional[str] = None      ):
           SCREAMING_SNAKE_CASE__       :  Any						     =		self._tokenizer.model.save(__UpperCAmelCase   ,   name=__UpperCAmelCase      )
           return tuple(__UpperCAmelCase      )














import unittest

from transformers import is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, require_tokenizers, require_torch, slow


if is_flax_available():
			import optax
			from flax.training.common_utils import onehot

			from transformers import AutoTokenizer, FlaxMTaForConditionalGeneration
			from transformers.models.ta.modeling_flax_ta import shift_tokens_right




@require_torch
@require_sentencepiece
@require_tokenizers
@require_flax
class 							snake_case_       (    unittest.TestCase      ):

						"""simple docstring"""



						@slow
						def        UpperCAmelCase__					(self:					Optional[int]   )    ->     List[str]:




											'''simple docstring'''



											__a					:						Optional[Any]      =					FlaxMTaForConditionalGeneration.from_pretrained("google/mt5-small"   )
											__a					:						Dict      =					AutoTokenizer.from_pretrained("google/mt5-small"   )

											__a					:						Any      =					tokenizer("Hello there" ,					return_tensors="np"   ).input_ids
											__a					:						Union[str, Any]      =					tokenizer("Hi I am" ,					return_tensors="np"   ).input_ids

											__a					:						Union[str, Any]      =					shift_tokens_right(__UpperCAmelCase ,					model.config.pad_token_id ,					model.config.decoder_start_token_id   )

											__a					:						Tuple      =					model(__UpperCAmelCase ,					decoder_input_ids=__UpperCAmelCase   ).logits
											__a					:						str      =					optax.softmax_cross_entropy(__UpperCAmelCase ,					onehot(__UpperCAmelCase ,					logits.shape[-1]   )   ).mean()

											__a					:						List[str]      =					-(labels.shape[-1] * loss.item())

											__a					:						Optional[int]      =					-84.91_27
											self.assertTrue(abs(mtf_score - EXPECTED_SCORE   ) < 1E-4   )





