






'''simple docstring'''





import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
						_UpperCamelCase						:						Dict  =					'pt'
elif is_tf_available():
						_UpperCamelCase						:						Tuple  =					'tf'
else:
						_UpperCamelCase						:						Optional[Any]  =					'jax'


class  snake_case__      (					a_     ,				unittest.TestCase):
					a_        =					PerceiverTokenizer
					a_        =					False




					def 		A      (  self  :		List[Any]      )    ->							int:
								super().setUp()
								UpperCAmelCase_      :       Tuple          =       PerceiverTokenizer()
								tokenizer.save_pretrained(self.tmpdirname      )




					@cached_property
					def 		A      (  self  :		Optional[int]      )    ->							Any:
								return PerceiverTokenizer.from_pretrained('''deepmind/language-perceiver'''      )




					def 		A      (  self  :		Any				, **_A  :		str      )    ->							PerceiverTokenizer:
								return self.tokenizer_class.from_pretrained(self.tmpdirname				, **lowerCAmelCase_      )




					def 		A      (  self  :		Dict				, _A  :		Dict				, _A  :		Union[str, Any]=False				, _A  :		Union[str, Any]=20				, _A  :		List[str]=5      )    ->							Tuple[str, list]:
								# XXX The default common tokenizer tests assume that every ID is decodable on its own.
								# This assumption is invalid for Perceiver because single bytes might not be
								# valid utf-8 (byte 128 for instance).
								# Here we're overriding the smallest possible method to provide
								# a clean sequence without making the same assumption.

								UpperCAmelCase_      :       Tuple          =       []
								for i in range(len(lowerCAmelCase_      )      ):
											try:
														UpperCAmelCase_      :       Tuple          =       tokenizer.decode([i]				, clean_up_tokenization_spaces=lowerCAmelCase_      )
											except UnicodeDecodeError:
														pass
											toks.append((i, tok)      )

								UpperCAmelCase_      :       List[str]          =       list(filter(lambda _A      : re.match(r'''^[ a-zA-Z]+$'''				, t[1]      )				, lowerCAmelCase_      )      )
								UpperCAmelCase_      :       Any          =       list(filter(lambda _A      : [t[0]] == tokenizer.encode(t[1]				, add_special_tokens=lowerCAmelCase_      )				, lowerCAmelCase_      )      )
								if max_length is not None and len(lowerCAmelCase_      ) > max_length:
											UpperCAmelCase_      :       List[Any]          =       toks[:max_length]
								if min_length is not None and len(lowerCAmelCase_      ) < min_length and len(lowerCAmelCase_      ) > 0:
											while len(lowerCAmelCase_      ) < min_length:
														UpperCAmelCase_      :       Optional[Any]          =       toks + toks
        # toks_str = [t[1] for t in toks]
								UpperCAmelCase_      :       List[Any]          =       [t[0] for t in toks]

								# Ensure consistency
								UpperCAmelCase_      :       Optional[Any]          =       tokenizer.decode(lowerCAmelCase_				, clean_up_tokenization_spaces=lowerCAmelCase_      )
								if " " not in output_txt and len(lowerCAmelCase_      ) > 1:
											UpperCAmelCase_      :       List[str]          =       (
											    tokenizer.decode([toks_ids[0]]				, clean_up_tokenization_spaces=lowerCAmelCase_      )
											    + ''' '''
											    + tokenizer.decode(toks_ids[1:]				, clean_up_tokenization_spaces=lowerCAmelCase_      )
											)
								if with_prefix_space:
											UpperCAmelCase_      :       List[str]          =       ''' ''' + output_txt
								UpperCAmelCase_      :       Any          =       tokenizer.encode(lowerCAmelCase_				, add_special_tokens=lowerCAmelCase_      )
								return output_txt, output_ids




					def 		A      (  self  :		Optional[Any]      )    ->							str:
								UpperCAmelCase_      :       Tuple          =       self.perceiver_tokenizer
								UpperCAmelCase_      :       Optional[Any]          =       '''Unicode €.'''
								UpperCAmelCase_      :       Optional[int]          =       tokenizer(lowerCAmelCase_      )
								UpperCAmelCase_      :       Optional[int]          =       [4, 91, 1_16, 1_11, 1_05, 1_17, 1_06, 1_07, 38, 2_32, 1_36, 1_78, 52, 5]
								self.assertEqual(encoded['''input_ids''']				, lowerCAmelCase_      )

								# decoding
								UpperCAmelCase_      :       Any          =       tokenizer.decode(lowerCAmelCase_      )
								self.assertEqual(lowerCAmelCase_				, '''[CLS]Unicode €.[SEP]'''      )

								UpperCAmelCase_      :       Optional[Any]          =       tokenizer('''e è é ê ë'''      )
								UpperCAmelCase_      :       Optional[int]          =       [4, 1_07, 38, 2_01, 1_74, 38, 2_01, 1_75, 38, 2_01, 1_76, 38, 2_01, 1_77, 5]
								self.assertEqual(encoded['''input_ids''']				, lowerCAmelCase_      )
								# decoding
								UpperCAmelCase_      :       Union[str, Any]          =       tokenizer.decode(lowerCAmelCase_      )
								self.assertEqual(lowerCAmelCase_				, '''[CLS]e è é ê ë[SEP]'''      )

								# encode/decode, but with `encode` instead of `__call__`
								self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''      )      )				, '''[CLS]e è é ê ë[SEP]'''      )




					def 		A      (  self  :		List[str]      )    ->							Tuple:
								UpperCAmelCase_      :       str          =       self.perceiver_tokenizer
								UpperCAmelCase_      :       Any          =       ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
								# fmt: off
								UpperCAmelCase_      :       Optional[int]          =       [4, 71, 38, 1_14, 1_17, 1_16, 1_09, 38, 1_18, 1_03, 1_20, 1_03, 1_09, 1_20, 1_03, 1_18, 1_10, 38, 1_08, 1_17, 1_20, 38, 1_21, 1_23, 1_15, 1_15, 1_03, 1_20, 1_11, 1_28, 1_03, 1_22, 1_11, 1_17, 1_16, 52, 5, 0]
								# fmt: on
								UpperCAmelCase_      :       List[str]          =       tokenizer(lowerCAmelCase_				, padding=lowerCAmelCase_				, return_tensors=lowerCAmelCase_      )
								self.assertIsInstance(lowerCAmelCase_				, lowerCAmelCase_      )

								if FRAMEWORK != "jax":
											UpperCAmelCase_      :       List[str]          =       list(batch.input_ids.numpy()[0]      )
								else:
											UpperCAmelCase_      :       Dict          =       list(batch.input_ids.tolist()[0]      )

								self.assertListEqual(lowerCAmelCase_				, lowerCAmelCase_      )

								self.assertEqual((2, 38)				, batch.input_ids.shape      )
								self.assertEqual((2, 38)				, batch.attention_mask.shape      )




					def 		A      (  self  :		List[Any]      )    ->							Optional[Any]:
								UpperCAmelCase_      :       Dict          =       self.perceiver_tokenizer
								UpperCAmelCase_      :       str          =       ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
								UpperCAmelCase_      :       Optional[int]          =       tokenizer(lowerCAmelCase_				, padding=lowerCAmelCase_				, return_tensors=lowerCAmelCase_      )
								# check if input_ids are returned and no decoder_input_ids
								self.assertIn('''input_ids'''				, lowerCAmelCase_      )
								self.assertIn('''attention_mask'''				, lowerCAmelCase_      )
								self.assertNotIn('''decoder_input_ids'''				, lowerCAmelCase_      )
								self.assertNotIn('''decoder_attention_mask'''				, lowerCAmelCase_      )




					def 		A      (  self  :		Optional[int]      )    ->							Dict:
								UpperCAmelCase_      :       Tuple          =       self.perceiver_tokenizer
								UpperCAmelCase_      :       Dict          =       [
								    '''Summary of the text.''',
								    '''Another summary.''',
								]
								UpperCAmelCase_      :       Any          =       tokenizer(
								    text_target=lowerCAmelCase_				, max_length=32				, padding='''max_length'''				, truncation=lowerCAmelCase_				, return_tensors=lowerCAmelCase_      )
								self.assertEqual(32				, targets['''input_ids'''].shape[1]      )




					def 		A      (  self  :		Any      )    ->							Tuple:
								# safety check on max_len default value so we are sure the test works
								UpperCAmelCase_      :       Optional[int]          =       self.get_tokenizers()
								for tokenizer in tokenizers:
											with self.subTest(F"{tokenizer.__class__.__name__}"      ):
														self.assertNotEqual(tokenizer.model_max_length				, 42      )

        # Now let's start the test
								UpperCAmelCase_      :       Optional[int]          =       self.get_tokenizers()
								for tokenizer in tokenizers:
											with self.subTest(F"{tokenizer.__class__.__name__}"      ):
														# Isolate this from the other tests because we save additional tokens/etc
														UpperCAmelCase_      :       List[Any]          =       tempfile.mkdtemp()

														UpperCAmelCase_      :       Dict          =       ''' He is very happy, UNwant\u00E9d,running'''
														UpperCAmelCase_      :       Dict          =       tokenizer.encode(lowerCAmelCase_				, add_special_tokens=lowerCAmelCase_      )
														tokenizer.save_pretrained(lowerCAmelCase_      )

														UpperCAmelCase_      :       Dict          =       tokenizer.__class__.from_pretrained(lowerCAmelCase_      )
														UpperCAmelCase_      :       Tuple          =       after_tokenizer.encode(lowerCAmelCase_				, add_special_tokens=lowerCAmelCase_      )
														self.assertListEqual(lowerCAmelCase_				, lowerCAmelCase_      )

														shutil.rmtree(lowerCAmelCase_      )

								UpperCAmelCase_      :       Optional[Any]          =       self.get_tokenizers(model_max_length=42      )
								for tokenizer in tokenizers:
											with self.subTest(F"{tokenizer.__class__.__name__}"      ):
														# Isolate this from the other tests because we save additional tokens/etc
														UpperCAmelCase_      :       Any          =       tempfile.mkdtemp()

														UpperCAmelCase_      :       Optional[int]          =       ''' He is very happy, UNwant\u00E9d,running'''
														tokenizer.add_tokens(['''bim''', '''bambam''']      )
														UpperCAmelCase_      :       Optional[int]          =       tokenizer.additional_special_tokens
														additional_special_tokens.append('''new_additional_special_token'''      )
														tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}      )
														UpperCAmelCase_      :       Tuple          =       tokenizer.encode(lowerCAmelCase_				, add_special_tokens=lowerCAmelCase_      )
														tokenizer.save_pretrained(lowerCAmelCase_      )

														UpperCAmelCase_      :       Any          =       tokenizer.__class__.from_pretrained(lowerCAmelCase_      )
														UpperCAmelCase_      :       Optional[Any]          =       after_tokenizer.encode(lowerCAmelCase_				, add_special_tokens=lowerCAmelCase_      )
														self.assertListEqual(lowerCAmelCase_				, lowerCAmelCase_      )
														self.assertIn('''new_additional_special_token'''				, after_tokenizer.additional_special_tokens      )
														self.assertEqual(after_tokenizer.model_max_length				, 42      )

														UpperCAmelCase_      :       Any          =       tokenizer.__class__.from_pretrained(lowerCAmelCase_				, model_max_length=43      )
														self.assertEqual(tokenizer.model_max_length				, 43      )

														shutil.rmtree(lowerCAmelCase_      )




					def 		A      (  self  :		Optional[Any]      )    ->							int:
								UpperCAmelCase_      :       int          =       []
								if self.test_slow_tokenizer:
											tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())      )

								if self.test_rust_tokenizer:
											tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())      )

								for tokenizer_class, tokenizer_utils in tokenizer_list:
											with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer_utils.save_pretrained(lowerCAmelCase_      )

														with open(os.path.join(lowerCAmelCase_				, '''special_tokens_map.json'''      )				, encoding='''utf-8'''      ) as json_file:
																	UpperCAmelCase_      :       Any          =       json.load(lowerCAmelCase_      )

														with open(os.path.join(lowerCAmelCase_				, '''tokenizer_config.json'''      )				, encoding='''utf-8'''      ) as json_file:
																	UpperCAmelCase_      :       str          =       json.load(lowerCAmelCase_      )

														UpperCAmelCase_      :       Any          =       [F"<extra_id_{i}>" for i in range(1_25      )]

														UpperCAmelCase_      :       List[Any]          =       added_tokens_extra_ids + [
														    '''an_additional_special_token'''
														]
														UpperCAmelCase_      :       Union[str, Any]          =       added_tokens_extra_ids + [
														    '''an_additional_special_token'''
														]

														with open(os.path.join(lowerCAmelCase_				, '''special_tokens_map.json'''      )				, '''w'''				, encoding='''utf-8'''      ) as outfile:
																	json.dump(lowerCAmelCase_				, lowerCAmelCase_      )
														with open(os.path.join(lowerCAmelCase_				, '''tokenizer_config.json'''      )				, '''w'''				, encoding='''utf-8'''      ) as outfile:
																	json.dump(lowerCAmelCase_				, lowerCAmelCase_      )

														# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
														# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
														# "special_tokens_map.json" files
														UpperCAmelCase_      :       Union[str, Any]          =       tokenizer_class.from_pretrained(
														    lowerCAmelCase_				, )
														self.assertIn(
														    '''an_additional_special_token'''				, tokenizer_without_change_in_init.additional_special_tokens      )
														self.assertEqual(
														    ['''an_additional_special_token''']				, tokenizer_without_change_in_init.convert_ids_to_tokens(
														        tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']      )      )				, )

														# Now we test that we can change the value of additional_special_tokens in the from_pretrained
														UpperCAmelCase_      :       Dict          =       added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''				, lstrip=lowerCAmelCase_      )]
														UpperCAmelCase_      :       List[str]          =       tokenizer_class.from_pretrained(
														    lowerCAmelCase_				, additional_special_tokens=lowerCAmelCase_				, )

														self.assertIn('''a_new_additional_special_token'''				, tokenizer.additional_special_tokens      )
														self.assertEqual(
														    ['''a_new_additional_special_token''']				, tokenizer.convert_ids_to_tokens(
														        tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']      )      )				, )




					def 		A      (  self  :		Optional[Any]      )    ->							int:
								UpperCAmelCase_      :       List[str]          =       self.perceiver_tokenizer
								self.assertEqual(tokenizer.decode([1_78]      )				, '''�'''      )




					def 		A      (  self  :		Optional[Any]      )    ->							str:
								pass




					def 		A      (  self  :		Tuple      )    ->							Optional[int]:
								pass




					def 		A      (  self  :		Union[str, Any]      )    ->							Any:
								pass




					def 		A      (  self  :		List[Any]      )    ->							Optional[int]:
								pass





					def 		A      (  self  :		Optional[int]      )    ->							Union[str, Any]:
								# The default common tokenizer tests uses invalid tokens for Perceiver that can only accept one-character
								# strings and special added tokens as tokens
								UpperCAmelCase_      :       List[Any]          =       self.get_tokenizers(fast=lowerCAmelCase_				, do_lower_case=lowerCAmelCase_      )
								for tokenizer in tokenizers:
											with self.subTest(F"{tokenizer.__class__.__name__}"      ):
														UpperCAmelCase_      :       Tuple          =       ['''[CLS]''', '''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''s''', '''t''', '''[SEP]''']
														UpperCAmelCase_      :       List[Any]          =       tokenizer.convert_tokens_to_string(lowerCAmelCase_      )

														self.assertIsInstance(lowerCAmelCase_				, lowerCAmelCase_      )






'''simple docstring'''





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCAmelCase__  			=				logging.get_logger(__name__)

UpperCAmelCase__  			=				{
    '''facebook/xlm-roberta-xl''': '''https://huggingface.co/facebook/xlm-roberta-xl/resolve/main/config.json''',
    '''facebook/xlm-roberta-xxl''': '''https://huggingface.co/facebook/xlm-roberta-xxl/resolve/main/config.json''',
    # See all XLM-RoBERTa-XL models at https://huggingface.co/models?filter=xlm-roberta-xl
}
class 						a__					(					a_   ):





			'''simple docstring'''

			A						:				Tuple					      =			'''xlm-roberta-xl'''
			def __init__(  self							:			Union[str, Any]			,    lowerCAmelCase_							:			Tuple=250_880			,    lowerCAmelCase_							:			Optional[int]=2_560			,    lowerCAmelCase_							:			Optional[Any]=36			,    lowerCAmelCase_							:			List[str]=32			,    lowerCAmelCase_							:			Optional[Any]=10_240			,    lowerCAmelCase_							:			Union[str, Any]="gelu"			,    lowerCAmelCase_							:			Optional[Any]=0.1			,    lowerCAmelCase_							:			Optional[Any]=0.1			,    lowerCAmelCase_							:			Optional[int]=514			,    lowerCAmelCase_							:			List[str]=1			,    lowerCAmelCase_							:			Optional[Any]=0.02			,    lowerCAmelCase_							:			List[str]=1E-05			,    lowerCAmelCase_							:			int=1			,    lowerCAmelCase_							:			List[str]=0			,    lowerCAmelCase_							:			List[Any]=2			,    lowerCAmelCase_							:			Optional[int]="absolute"			,    lowerCAmelCase_							:			List[Any]=True			,    lowerCAmelCase_							:			Tuple=None			,    **lowerCAmelCase_							:			List[str]			,    )      ->		str:
						super().__init__(pad_token_id=lowerCAmelCase_			,    bos_token_id=lowerCAmelCase_			,    eos_token_id=lowerCAmelCase_			,    **lowerCAmelCase_	)
						__A=	vocab_size
						__A=	hidden_size
						__A=	num_hidden_layers
						__A=	num_attention_heads
						__A=	hidden_act
						__A=	intermediate_size
						__A=	hidden_dropout_prob
						__A=	attention_probs_dropout_prob
						__A=	max_position_embeddings
						__A=	type_vocab_size
						__A=	initializer_range
						__A=	layer_norm_eps
						__A=	position_embedding_type
						__A=	use_cache
						__A=	classifier_dropout


class 						a__					(					a_   ):





			'''simple docstring'''

			@property
			def 					lowerCAmelCase     (  self							:			Optional[int]	)      ->		Mapping[str, Mapping[int, str]]:
						if self.task == "multiple-choice":
									__A=	{0: 'batch', 1: 'choice', 2: 'sequence'}
						else:
									__A=	{0: 'batch', 1: 'sequence'}
						return OrderedDict(
						    [
						        ('input_ids', dynamic_axis),
						        ('attention_mask', dynamic_axis),
						    ]	)



