





"""simple docstring"""


import unittest

from transformers import MPNetConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MPNetForMaskedLM,
		    MPNetForMultipleChoice,
		    MPNetForQuestionAnswering,
		    MPNetForSequenceClassification,
		    MPNetForTokenClassification,
		    MPNetModel,
		)






class 							lowercase__			:

			"""simple docstring"""







			def __init__(      self						,   _A						,   _A=1_3						,   _A=7						,   _A=True						,   _A=True						,   _A=False						,   _A=True						,   _A=9_9						,   _A=6_4						,   _A=5						,   _A=4						,   _A=6_4						,   _A="gelu"						,   _A=0.1						,   _A=0.1						,   _A=5_1_2						,   _A=1_6						,   _A=2						,   _A=0.02						,   _A=3						,   _A=4						,   _A=None						,   ):






									'''simple docstring'''







									UpperCamelCase				:		Union[str, Any]				   =			parent
									UpperCamelCase				:		Tuple				   =			batch_size
									UpperCamelCase				:		Any				   =			seq_length
									UpperCamelCase				:		str				   =			is_training
									UpperCamelCase				:		Optional[Any]				   =			use_input_mask
									UpperCamelCase				:		Tuple				   =			use_token_type_ids
									UpperCamelCase				:		List[str]				   =			use_labels
									UpperCamelCase				:		str				   =			vocab_size
									UpperCamelCase				:		Union[str, Any]				   =			hidden_size
									UpperCamelCase				:		Dict				   =			num_hidden_layers
									UpperCamelCase				:		Union[str, Any]				   =			num_attention_heads
									UpperCamelCase				:		Any				   =			intermediate_size
									UpperCamelCase				:		Any				   =			hidden_act
									UpperCamelCase				:		Dict				   =			hidden_dropout_prob
									UpperCamelCase				:		Optional[Any]				   =			attention_probs_dropout_prob
									UpperCamelCase				:		Any				   =			max_position_embeddings
									UpperCamelCase				:		Dict				   =			type_vocab_size
									UpperCamelCase				:		Dict				   =			type_sequence_label_size
									UpperCamelCase				:		Tuple				   =			initializer_range
									UpperCamelCase				:		str				   =			num_labels
									UpperCamelCase				:		Optional[Any]				   =			num_choices
									UpperCamelCase				:		Optional[int]				   =			scope







			def   _a      (      self    ):






									'''simple docstring'''







									return MPNetConfig.from_pretrained("""microsoft/mpnet-base"""    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		List[str]				   =			ids_tensor([self.batch_size, self.seq_length]						,   self.vocab_size    )

									UpperCamelCase				:		Optional[Any]				   =			None
									if self.use_input_mask:
															UpperCamelCase				:		int				   =			random_attention_mask([self.batch_size, self.seq_length]    )

									UpperCamelCase				:		List[str]				   =			None
									UpperCamelCase				:		Dict				   =			None
									UpperCamelCase				:		Union[str, Any]				   =			None
									if self.use_labels:
															UpperCamelCase				:		Optional[int]				   =			ids_tensor([self.batch_size]						,   self.type_sequence_label_size    )
															UpperCamelCase				:		Optional[int]				   =			ids_tensor([self.batch_size, self.seq_length]						,   self.num_labels    )
															UpperCamelCase				:		List[str]				   =			ids_tensor([self.batch_size]						,   self.num_choices    )

									UpperCamelCase				:		Optional[Any]				   =			self.get_config()
									return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels







			def   _a      (      self    ):






									'''simple docstring'''







									return MPNetConfig(
									    vocab_size=self.vocab_size						,   hidden_size=self.hidden_size						,   num_hidden_layers=self.num_hidden_layers						,   num_attention_heads=self.num_attention_heads						,   intermediate_size=self.intermediate_size						,   hidden_act=self.hidden_act						,   hidden_dropout_prob=self.hidden_dropout_prob						,   attention_probs_dropout_prob=self.attention_probs_dropout_prob						,   max_position_embeddings=self.max_position_embeddings						,   initializer_range=self.initializer_range						,   )







			def   _a      (      self						,   _A						,   _A						,   _A						,   _A						,   _A						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		int				   =			MPNetModel(config=_A    )
									model.to(_A    )
									model.eval()
									UpperCamelCase				:		Union[str, Any]				   =			model(_A						,   _A    )
									UpperCamelCase				:		List[Any]				   =			model(_A    )
									self.parent.assertEqual(result.last_hidden_state.shape						,   (self.batch_size, self.seq_length, self.hidden_size)    )
									self.parent.assertEqual(result.pooler_output.shape						,   (self.batch_size, self.hidden_size)    )







			def   _a      (      self						,   _A						,   _A						,   _A						,   _A						,   _A						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		Tuple				   =			MPNetForQuestionAnswering(config=_A    )
									model.to(_A    )
									model.eval()
									UpperCamelCase				:		Tuple				   =			model(
									    _A						,   attention_mask=_A						,   start_positions=_A						,   end_positions=_A						,   )
									self.parent.assertEqual(result.start_logits.shape						,   (self.batch_size, self.seq_length)    )
									self.parent.assertEqual(result.end_logits.shape						,   (self.batch_size, self.seq_length)    )







			def   _a      (      self						,   _A						,   _A						,   _A						,   _A						,   _A						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[Any]				   =			self.num_labels
									UpperCamelCase				:		int				   =			MPNetForSequenceClassification(_A    )
									model.to(_A    )
									model.eval()
									UpperCamelCase				:		Optional[Any]				   =			model(_A						,   attention_mask=_A						,   labels=_A    )
									self.parent.assertEqual(result.logits.shape						,   (self.batch_size, self.num_labels)    )







			def   _a      (      self						,   _A						,   _A						,   _A						,   _A						,   _A						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[Any]				   =			self.num_choices
									UpperCamelCase				:		Optional[int]				   =			MPNetForMultipleChoice(config=_A    )
									model.to(_A    )
									model.eval()
									UpperCamelCase				:		str				   =			input_ids.unsqueeze(1    ).expand(-1						,   self.num_choices						,   -1    ).contiguous()
									UpperCamelCase				:		List[Any]				   =			input_mask.unsqueeze(1    ).expand(-1						,   self.num_choices						,   -1    ).contiguous()
									UpperCamelCase				:		Union[str, Any]				   =			model(
									    _A						,   attention_mask=_A						,   labels=_A						,   )
									self.parent.assertEqual(result.logits.shape						,   (self.batch_size, self.num_choices)    )







			def   _a      (      self						,   _A						,   _A						,   _A						,   _A						,   _A						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		Any				   =			self.num_labels
									UpperCamelCase				:		List[Any]				   =			MPNetForTokenClassification(config=_A    )
									model.to(_A    )
									model.eval()
									UpperCamelCase				:		Any				   =			model(_A						,   attention_mask=_A						,   labels=_A    )
									self.parent.assertEqual(result.logits.shape						,   (self.batch_size, self.seq_length, self.num_labels)    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Any				   =			self.prepare_config_and_inputs()
									((UpperCamelCase)   ,						(UpperCamelCase)   ,						(UpperCamelCase)   ,						(UpperCamelCase)   ,						(UpperCamelCase)   ,						(UpperCamelCase))				:		List[Any]				   =			config_and_inputs
									UpperCamelCase				:		str				   =			{"""input_ids""": input_ids, """attention_mask""": input_mask}
									return config, inputs_dict






@require_torch
class 							lowercase__			(   __SCREAMING_SNAKE_CASE					,     __SCREAMING_SNAKE_CASE					,     unittest.TestCase	):

			"""simple docstring"""
			__lowerCAmelCase     :				Tuple										=  (
			    (
			        MPNetForMaskedLM,
			        MPNetForMultipleChoice,
			        MPNetForQuestionAnswering,
			        MPNetForSequenceClassification,
			        MPNetForTokenClassification,
			        MPNetModel,
			    )
			    if is_torch_available()
			    else ()
			)
			__lowerCAmelCase     :				Any										=  (
			    {
			        """feature-extraction""": MPNetModel,
			        """fill-mask""": MPNetForMaskedLM,
			        """question-answering""": MPNetForQuestionAnswering,
			        """text-classification""": MPNetForSequenceClassification,
			        """token-classification""": MPNetForTokenClassification,
			        """zero-shot""": MPNetForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			__lowerCAmelCase     :				Union[str, Any]										=  False
			__lowerCAmelCase     :				int										=  True







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[Any]				   =			MPNetModelTester(self    )
									UpperCamelCase				:		Optional[int]				   =			ConfigTester(self						,   config_class=_A						,   hidden_size=3_7    )







			def   _a      (      self    ):






									'''simple docstring'''







									self.config_tester.run_common_tests()







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		int				   =			self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_model(*_A    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		List[Any]				   =			self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_for_sequence_classification(*_A    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		List[Any]				   =			self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_for_multiple_choice(*_A    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Dict				   =			self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_for_token_classification(*_A    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Any				   =			self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_mpnet_for_question_answering(*_A    )






@require_torch
class 							lowercase__			(   unittest.TestCase	):

			"""simple docstring"""







			@slow
			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Dict				   =			MPNetModel.from_pretrained("""microsoft/mpnet-base"""    )
									UpperCamelCase				:		Tuple				   =			torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]    )
									UpperCamelCase				:		Any				   =			model(_A    )[0]
									UpperCamelCase				:		Tuple				   =			torch.Size((1, 1_1, 7_6_8)    )
									self.assertEqual(output.shape						,   _A    )
									UpperCamelCase				:		Optional[Any]				   =			torch.tensor(
									    [[[-0.05_50, 0.19_43, -0.07_40], [-0.05_62, 0.22_11, -0.05_79], [-0.04_37, 0.33_37, -0.06_41]]]    )
									# compare the actual values for a slice.
									self.assertTrue(torch.allclose(output[:, :3, :3]						,   _A						,   atol=1e-4    )    )








"""simple docstring"""






import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
     snake_case   					=				'''pt'''
elif is_tf_available():
     snake_case   					=				'''tf'''
else:
     snake_case   					=				'''jax'''



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE,unittest.TestCase  ):
   A__		:	Union[str, Any]           =  PerceiverTokenizer
   A__		:	Optional[int]           =  False



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     super().setUp()
     _snake_case   	=      PerceiverTokenizer()
     tokenizer.save_pretrained(self.tmpdirname				)



   @cached_property
   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     return PerceiverTokenizer.from_pretrained('''deepmind/language-perceiver'''				)



   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       **__lowerCamelCase    :	Union[str, Any]				):



     """simple docstring"""
     return self.tokenizer_class.from_pretrained(self.tmpdirname							,       **__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	List[Any]							,       __lowerCamelCase    :	List[str]=False							,       __lowerCamelCase    :	int=2_0							,       __lowerCamelCase    :	List[Any]=5				):



     """simple docstring"""
     # XXX The default common tokenizer tests assume that every ID is decodable on its own.
     # This assumption is invalid for Perceiver because single bytes might not be
     # valid utf-8 (byte 128 for instance).
     # Here we're overriding the smallest possible method to provide
     # a clean sequence without making the same assumption.

     _snake_case   	=      []
     for i in range(len(__lowerCamelCase				)				):
       try:
         _snake_case   	=      tokenizer.decode([i]							,       clean_up_tokenization_spaces=__lowerCamelCase				)
       except UnicodeDecodeError:
         pass
       toks.append((i, tok)				)

     _snake_case   	=      list(filter(lambda __lowerCamelCase				: re.match(R'''^[ a-zA-Z]+$'''							,       t[1]				)							,       __lowerCamelCase				)				)
     _snake_case   	=      list(filter(lambda __lowerCamelCase				: [t[0]] == tokenizer.encode(t[1]							,       add_special_tokens=__lowerCamelCase				)							,       __lowerCamelCase				)				)
     if max_length is not None and len(__lowerCamelCase				) > max_length:
       _snake_case   	=      toks[:max_length]
     if min_length is not None and len(__lowerCamelCase				) < min_length and len(__lowerCamelCase				) > 0:
       while len(__lowerCamelCase				) < min_length:
         _snake_case   	=      toks + toks
        # toks_str = [t[1] for t in toks]
     _snake_case   	=      [t[0] for t in toks]

     # Ensure consistency
     _snake_case   	=      tokenizer.decode(__lowerCamelCase							,       clean_up_tokenization_spaces=__lowerCamelCase				)
     if " " not in output_txt and len(__lowerCamelCase				) > 1:
       _snake_case   	=      (
           tokenizer.decode([toks_ids[0]]							,       clean_up_tokenization_spaces=__lowerCamelCase				)
           + ''' '''
           + tokenizer.decode(toks_ids[1:]							,       clean_up_tokenization_spaces=__lowerCamelCase				)
       )
     if with_prefix_space:
       _snake_case   	=      ''' ''' + output_txt
     _snake_case   	=      tokenizer.encode(__lowerCamelCase							,       add_special_tokens=__lowerCamelCase				)
     return output_txt, output_ids



   def    __UpperCAmelCase    (       self    :	List[str]				):



     """simple docstring"""
     _snake_case   	=      self.perceiver_tokenizer
     _snake_case   	=      '''Unicode €.'''
     _snake_case   	=      tokenizer(__lowerCamelCase				)
     _snake_case   	=      [4, 9_1, 1_1_6, 1_1_1, 1_0_5, 1_1_7, 1_0_6, 1_0_7, 3_8, 2_3_2, 1_3_6, 1_7_8, 5_2, 5]
     self.assertEqual(encoded['''input_ids''']							,       __lowerCamelCase				)

     # decoding
     _snake_case   	=      tokenizer.decode(__lowerCamelCase				)
     self.assertEqual(__lowerCamelCase							,       '''[CLS]Unicode €.[SEP]'''				)

     _snake_case   	=      tokenizer('''e è é ê ë'''				)
     _snake_case   	=      [4, 1_0_7, 3_8, 2_0_1, 1_7_4, 3_8, 2_0_1, 1_7_5, 3_8, 2_0_1, 1_7_6, 3_8, 2_0_1, 1_7_7, 5]
     self.assertEqual(encoded['''input_ids''']							,       __lowerCamelCase				)
     # decoding
     _snake_case   	=      tokenizer.decode(__lowerCamelCase				)
     self.assertEqual(__lowerCamelCase							,       '''[CLS]e è é ê ë[SEP]'''				)

     # encode/decode, but with `encode` instead of `__call__`
     self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''				)				)							,       '''[CLS]e è é ê ë[SEP]'''				)



   def    __UpperCAmelCase    (       self    :	Optional[int]				):



     """simple docstring"""
     _snake_case   	=      self.perceiver_tokenizer
     _snake_case   	=      ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
     # fmt: off
     _snake_case   	=      [4, 7_1, 3_8, 1_1_4, 1_1_7, 1_1_6, 1_0_9, 3_8, 1_1_8, 1_0_3, 1_2_0, 1_0_3, 1_0_9, 1_2_0, 1_0_3, 1_1_8, 1_1_0, 3_8, 1_0_8, 1_1_7, 1_2_0, 3_8, 1_2_1, 1_2_3, 1_1_5, 1_1_5, 1_0_3, 1_2_0, 1_1_1, 1_2_8, 1_0_3, 1_2_2, 1_1_1, 1_1_7, 1_1_6, 5_2, 5, 0]
     # fmt: on
     _snake_case   	=      tokenizer(__lowerCamelCase							,       padding=__lowerCamelCase							,       return_tensors=__lowerCamelCase				)
     self.assertIsInstance(__lowerCamelCase							,       __lowerCamelCase				)

     if FRAMEWORK != "jax":
       _snake_case   	=      list(batch.input_ids.numpy()[0]				)
     else:
       _snake_case   	=      list(batch.input_ids.tolist()[0]				)

     self.assertListEqual(__lowerCamelCase							,       __lowerCamelCase				)

     self.assertEqual((2, 3_8)							,       batch.input_ids.shape				)
     self.assertEqual((2, 3_8)							,       batch.attention_mask.shape				)



   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     _snake_case   	=      self.perceiver_tokenizer
     _snake_case   	=      ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
     _snake_case   	=      tokenizer(__lowerCamelCase							,       padding=__lowerCamelCase							,       return_tensors=__lowerCamelCase				)
     # check if input_ids are returned and no decoder_input_ids
     self.assertIn('''input_ids'''							,       __lowerCamelCase				)
     self.assertIn('''attention_mask'''							,       __lowerCamelCase				)
     self.assertNotIn('''decoder_input_ids'''							,       __lowerCamelCase				)
     self.assertNotIn('''decoder_attention_mask'''							,       __lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     _snake_case   	=      self.perceiver_tokenizer
     _snake_case   	=      [
         '''Summary of the text.''',
         '''Another summary.''',
     ]
     _snake_case   	=      tokenizer(
         text_target=__lowerCamelCase							,       max_length=3_2							,       padding='''max_length'''							,       truncation=__lowerCamelCase							,       return_tensors=__lowerCamelCase				)
     self.assertEqual(3_2							,       targets['''input_ids'''].shape[1]				)



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     # safety check on max_len default value so we are sure the test works
     _snake_case   	=      self.get_tokenizers()
     for tokenizer in tokenizers:
       with self.subTest(f"""{tokenizer.__class__.__name__}"""				):
         self.assertNotEqual(tokenizer.model_max_length							,       4_2				)

        # Now let's start the test
     _snake_case   	=      self.get_tokenizers()
     for tokenizer in tokenizers:
       with self.subTest(f"""{tokenizer.__class__.__name__}"""				):
         # Isolate this from the other tests because we save additional tokens/etc
         _snake_case   	=      tempfile.mkdtemp()

         _snake_case   	=      ''' He is very happy, UNwant\u00E9d,running'''
         _snake_case   	=      tokenizer.encode(__lowerCamelCase							,       add_special_tokens=__lowerCamelCase				)
         tokenizer.save_pretrained(__lowerCamelCase				)

         _snake_case   	=      tokenizer.__class__.from_pretrained(__lowerCamelCase				)
         _snake_case   	=      after_tokenizer.encode(__lowerCamelCase							,       add_special_tokens=__lowerCamelCase				)
         self.assertListEqual(__lowerCamelCase							,       __lowerCamelCase				)

         shutil.rmtree(__lowerCamelCase				)

     _snake_case   	=      self.get_tokenizers(model_max_length=4_2				)
     for tokenizer in tokenizers:
       with self.subTest(f"""{tokenizer.__class__.__name__}"""				):
         # Isolate this from the other tests because we save additional tokens/etc
         _snake_case   	=      tempfile.mkdtemp()

         _snake_case   	=      ''' He is very happy, UNwant\u00E9d,running'''
         tokenizer.add_tokens(['''bim''', '''bambam''']				)
         _snake_case   	=      tokenizer.additional_special_tokens
         additional_special_tokens.append('''new_additional_special_token'''				)
         tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}				)
         _snake_case   	=      tokenizer.encode(__lowerCamelCase							,       add_special_tokens=__lowerCamelCase				)
         tokenizer.save_pretrained(__lowerCamelCase				)

         _snake_case   	=      tokenizer.__class__.from_pretrained(__lowerCamelCase				)
         _snake_case   	=      after_tokenizer.encode(__lowerCamelCase							,       add_special_tokens=__lowerCamelCase				)
         self.assertListEqual(__lowerCamelCase							,       __lowerCamelCase				)
         self.assertIn('''new_additional_special_token'''							,       after_tokenizer.additional_special_tokens				)
         self.assertEqual(after_tokenizer.model_max_length							,       4_2				)

         _snake_case   	=      tokenizer.__class__.from_pretrained(__lowerCamelCase							,       model_max_length=4_3				)
         self.assertEqual(tokenizer.model_max_length							,       4_3				)

         shutil.rmtree(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Tuple				):



     """simple docstring"""
     _snake_case   	=      []
     if self.test_slow_tokenizer:
       tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())				)

     if self.test_rust_tokenizer:
       tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())				)

     for tokenizer_class, tokenizer_utils in tokenizer_list:
       with tempfile.TemporaryDirectory() as tmp_dir:
         tokenizer_utils.save_pretrained(__lowerCamelCase				)

         with open(os.path.join(__lowerCamelCase							,       '''special_tokens_map.json'''				)							,       encoding='''utf-8'''				) as json_file:
           _snake_case   	=      json.load(__lowerCamelCase				)

         with open(os.path.join(__lowerCamelCase							,       '''tokenizer_config.json'''				)							,       encoding='''utf-8'''				) as json_file:
           _snake_case   	=      json.load(__lowerCamelCase				)

         _snake_case   	=      [f"""<extra_id_{i}>""" for i in range(1_2_5				)]

         _snake_case   	=      added_tokens_extra_ids + [
             '''an_additional_special_token'''
         ]
         _snake_case   	=      added_tokens_extra_ids + [
             '''an_additional_special_token'''
         ]

         with open(os.path.join(__lowerCamelCase							,       '''special_tokens_map.json'''				)							,       '''w'''							,       encoding='''utf-8'''				) as outfile:
           json.dump(__lowerCamelCase							,       __lowerCamelCase				)
         with open(os.path.join(__lowerCamelCase							,       '''tokenizer_config.json'''				)							,       '''w'''							,       encoding='''utf-8'''				) as outfile:
           json.dump(__lowerCamelCase							,       __lowerCamelCase				)

         # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
         # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
         # "special_tokens_map.json" files
         _snake_case   	=      tokenizer_class.from_pretrained(
             __lowerCamelCase							,       )
         self.assertIn(
             '''an_additional_special_token'''							,       tokenizer_without_change_in_init.additional_special_tokens				)
         self.assertEqual(
             ['''an_additional_special_token''']							,       tokenizer_without_change_in_init.convert_ids_to_tokens(
                 tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']				)				)							,       )

         # Now we test that we can change the value of additional_special_tokens in the from_pretrained
         _snake_case   	=      added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''							,       lstrip=__lowerCamelCase				)]
         _snake_case   	=      tokenizer_class.from_pretrained(
             __lowerCamelCase							,       additional_special_tokens=__lowerCamelCase							,       )

         self.assertIn('''a_new_additional_special_token'''							,       tokenizer.additional_special_tokens				)
         self.assertEqual(
             ['''a_new_additional_special_token''']							,       tokenizer.convert_ids_to_tokens(
                 tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']				)				)							,       )



   def    __UpperCAmelCase    (       self    :	Optional[int]				):



     """simple docstring"""
     _snake_case   	=      self.perceiver_tokenizer
     self.assertEqual(tokenizer.decode([1_7_8]				)							,       '''�'''				)



   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     pass



   def    __UpperCAmelCase    (       self    :	str				):



     """simple docstring"""
     pass



   def    __UpperCAmelCase    (       self    :	str				):



     """simple docstring"""
     pass



   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     pass





   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     # The default common tokenizer tests uses invalid tokens for Perceiver that can only accept one-character
     # strings and special added tokens as tokens
     _snake_case   	=      self.get_tokenizers(fast=__lowerCamelCase							,       do_lower_case=__lowerCamelCase				)
     for tokenizer in tokenizers:
       with self.subTest(f"""{tokenizer.__class__.__name__}"""				):
         _snake_case   	=      ['''[CLS]''', '''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''s''', '''t''', '''[SEP]''']
         _snake_case   	=      tokenizer.convert_tokens_to_string(__lowerCamelCase				)

         self.assertIsInstance(__lowerCamelCase							,       __lowerCamelCase				)





