


'''simple docstring'''







import math
import os
import re
import sys
import unittest
from pathlib import Path
from typing import Tuple
from unittest.mock import patch

from parameterized import parameterized

from transformers.testing_utils import (
    CaptureStderr,
    ExtendSysPath,
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_torch_dist_unique_port,
    require_apex,
    require_bitsandbytes,
    require_fairscale,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    require_torch_non_multi_gpu,
    slow,
)
from transformers.trainer_callback import TrainerState
from transformers.trainer_utils import set_seed


_SCREAMING_SNAKE_CASE							: str										=			os.path.abspath(os.path.dirname(__file__))
with ExtendSysPath(f"{bindir}/../../examples/pytorch/translation"):
		from run_translation import main  # noqa


set_seed(42)
_SCREAMING_SNAKE_CASE							: List[Any]										=			"sshleifer/student_marian_en_ro_6_1"
_SCREAMING_SNAKE_CASE							: Any										=			"sshleifer/tiny-mbart"






@require_torch
class 		_snake_case     (					__lowerCamelCase		):






						'''simple docstring'''







						def lowerCAmelCase__			(	self:						int   ,				__UpperCamelCase:						Optional[int]=False   ,				__UpperCamelCase:						Union[str, Any]=None   ,				__UpperCamelCase:						Union[str, Any]=True   ,				__UpperCamelCase:						Optional[int]=True   ,				__UpperCamelCase:						List[str]=True   ,				__UpperCamelCase:						List[Any]=True   ,				)				->						str:
										__magic_name__					:       Dict    							=							self.run_trainer(
										    eval_steps=1   ,				max_len=12   ,				model_name=lowerCamelCase__   ,				num_train_epochs=1   ,				distributed=lowerCamelCase__   ,				extra_args_str=lowerCamelCase__   ,				predict_with_generate=lowerCamelCase__   ,				do_train=lowerCamelCase__   ,				do_eval=lowerCamelCase__   ,				do_predict=lowerCamelCase__   ,				)
										__magic_name__					:       Any    							=							TrainerState.load_from_json(os.path.join(lowerCamelCase__   ,				"trainer_state.json"  )  ).log_history

										if not do_eval:
														return

										__magic_name__					:       Dict    							=							[log for log in logs if "eval_loss" in log.keys()]

										__magic_name__					:       Tuple    							=							eval_metrics[0]
										if predict_with_generate:
														assert "eval_bleu" in first_step_stats

														__magic_name__					:       Dict    							=							eval_metrics[-1]
														assert isinstance(last_step_stats["eval_bleu"]   ,				lowerCamelCase__  )
														assert not math.isnan(float(last_step_stats["eval_loss"]  )  ), "eval_loss must not be `nan`"







						@require_torch_non_multi_gpu
						def lowerCAmelCase__			(	self:						List[str]  )				->						Optional[Any]:
										self.run_seqaseq_quick()







						@require_torch_multi_gpu
						def lowerCAmelCase__			(	self:						int  )				->						Optional[Any]:
										self.run_seqaseq_quick(distributed=lowerCamelCase__  )







						@require_torch_multi_gpu
						def lowerCAmelCase__			(	self:						List[str]  )				->						List[Any]:
										self.run_seqaseq_quick(distributed=lowerCamelCase__  )







						@unittest.skip("Requires an update of the env running those tests"  )
						@require_torch_multi_gpu
						@require_fairscale
						def lowerCAmelCase__			(	self:						List[Any]  )				->						Optional[Any]:
										self.run_seqaseq_quick(distributed=lowerCamelCase__   ,				extra_args_str="--sharded_ddp simple"  )







						@unittest.skip("Requires an update of the env running those tests"  )
						@require_torch_multi_gpu
						@require_fairscale
						def lowerCAmelCase__			(	self:						List[str]  )				->						str:
										self.run_seqaseq_quick(distributed=lowerCamelCase__   ,				extra_args_str="--sharded_ddp simple --fp16"  )







						@unittest.skip("Requires an update of the env running those tests"  )
						@require_torch_multi_gpu
						@require_fairscale
						def lowerCAmelCase__			(	self:						str  )				->						Dict:
										self.run_seqaseq_quick(distributed=lowerCamelCase__   ,				extra_args_str="--sharded_ddp zero_dp_2"   ,				predict_with_generate=lowerCamelCase__  )







						@unittest.skip("Requires an update of the env running those tests"  )
						@require_torch_multi_gpu
						@require_fairscale
						def lowerCAmelCase__			(	self:						Union[str, Any]  )				->						Dict:
										self.run_seqaseq_quick(
										    distributed=lowerCamelCase__   ,				extra_args_str="--sharded_ddp zero_dp_2 --fp16"   ,				predict_with_generate=lowerCamelCase__  )







						@require_apex
						@require_torch_gpu
						def lowerCAmelCase__			(	self:						List[Any]  )				->						Tuple:
										self.run_seqaseq_quick(distributed=lowerCamelCase__   ,				extra_args_str="--fp16 --fp16_backend=apex"  )
										# test 2nd time - was getting eval_loss': nan'
										# to reproduce the problem set distributed=False
										self.run_seqaseq_quick(distributed=lowerCamelCase__   ,				extra_args_str="--fp16 --fp16_backend=apex"  )







						@parameterized.expand(["base", "low", "high", "mixed"]  )
						@require_torch_multi_gpu
						def lowerCAmelCase__			(	self:						int   ,				__UpperCamelCase:						Tuple  )				->						Optional[Any]:
										__magic_name__					:       List[Any]    							=							{
										    # test with the default log_level - should be info and thus log info once
										    "base": {"extra_args_str": "", "n_matches": 1},
										    # test with low log_level and log_level_replica - should be noisy on all processes
										    # now the info string should appear twice on 2 processes
										    "low": {"extra_args_str": "--log_level debug --log_level_replica debug", "n_matches": 2},
										    # test with high log_level and low log_level_replica
										    # now the info string should appear once only on the replica
										    "high": {"extra_args_str": "--log_level error --log_level_replica debug", "n_matches": 1},
										    # test with high log_level and log_level_replica - should be quiet on all processes
										    "mixed": {"extra_args_str": "--log_level error --log_level_replica error", "n_matches": 0},
										}

										__magic_name__					:       List[Any]    							=							experiments[experiment_id]
										__magic_name__					:       Optional[Any]    							=							{"distributed": True, "predict_with_generate": False, "do_eval": False, "do_predict": False}
										__magic_name__					:       int    							=							"Running training"
										with CaptureStderr() as cl:
														self.run_seqaseq_quick(**lowerCamelCase__   ,				extra_args_str=data["extra_args_str"]  )
										__magic_name__					:       Optional[int]    							=							len(re.findall(lowerCamelCase__   ,				cl.err  )  )
										self.assertEqual(lowerCamelCase__   ,				data["n_matches"]  )







						@slow
						def lowerCAmelCase__			(	self:						Tuple  )				->						List[str]:
										__magic_name__					:       List[str]    							=							self.run_trainer(
										    eval_steps=2   ,				max_len=128   ,				model_name=lowerCamelCase__   ,				learning_rate=3E-4   ,				num_train_epochs=10   ,				distributed=lowerCamelCase__   ,				)

										# Check metrics
										__magic_name__					:       Optional[Any]    							=							TrainerState.load_from_json(os.path.join(lowerCamelCase__   ,				"trainer_state.json"  )  ).log_history
										__magic_name__					:       List[Any]    							=							[log for log in logs if "eval_loss" in log.keys()]
										__magic_name__					:       Optional[Any]    							=							eval_metrics[0]
										__magic_name__					:       Tuple    							=							eval_metrics[-1]

										assert first_step_stats["eval_loss"] > last_step_stats["eval_loss"], "model learned nothing"
										assert isinstance(last_step_stats["eval_bleu"]   ,				lowerCamelCase__  )

										# test if do_predict saves generations and metrics
										__magic_name__					:       List[str]    							=							os.listdir(lowerCamelCase__  )
										__magic_name__					:       Optional[Any]    							=							{os.path.basename(lowerCamelCase__  ) for p in contents}
										assert "generated_predictions.txt" in contents
										assert "predict_results.json" in contents







						@slow
						@require_bitsandbytes
						def lowerCAmelCase__			(	self:						str  )				->						List[str]:
										from transformers.training_args import OptimizerNames

										def train_and_return_metrics(__UpperCamelCase:						str  ) -> Tuple[int, float]:
														__magic_name__					:       Optional[int]    							=							"--skip_memory_metrics 0"

														__magic_name__					:       List[Any]    							=							self.run_trainer(
														    max_len=128   ,				model_name=lowerCamelCase__   ,				learning_rate=3E-4   ,				num_train_epochs=1   ,				optim=lowerCamelCase__   ,				distributed=lowerCamelCase__   ,				extra_args_str=lowerCamelCase__   ,				do_eval=lowerCamelCase__   ,				do_predict=lowerCamelCase__   ,				n_gpus_to_use=1   ,				)

														# Check metrics
														__magic_name__					:       Dict    							=							TrainerState.load_from_json(Path(lowerCamelCase__   ,				"trainer_state.json"  )  ).log_history
														__magic_name__					:       Optional[int]    							=							int(logs[0]["train_mem_gpu_peaked_delta"] / 2**20  )
														__magic_name__					:       Tuple    							=							int(logs[0]["train_mem_gpu_alloc_delta"] / 2**20  )

														__magic_name__					:       Union[str, Any]    							=							logs[0]["train_loss"]
														return gpu_peak_mem_mb, gpu_alloc_mem_mb, loss

										__magic_name__							,  __magic_name__							,  __magic_name__					:       Tuple    							=							train_and_return_metrics(OptimizerNames.ADAMW_TORCH.value  )
										__magic_name__							,  __magic_name__							,  __magic_name__					:       Tuple    							=							train_and_return_metrics(OptimizerNames.ADAMW_BNB.value  )

										__magic_name__					:       Tuple    							=							gpu_alloc_mem_orig - gpu_alloc_mem_bnb

										__magic_name__					:       str    							=							gpu_peak_mem_orig + gpu_alloc_mem_orig
										__magic_name__					:       Tuple    							=							gpu_peak_mem_bnb + gpu_alloc_mem_bnb
										__magic_name__					:       Any    							=							gpu_total_mem_orig - gpu_total_mem_bnb

										# sshleifer/student_marian_en_ro_6_1 has 54M parameter, 29M of which is `nn.Embedding` which
										# doesn't get quantized and remains in fp32. Therefore we only have 25M parameters quantized
										# in 2 bytes and the diff in optim memory usage is derived as so:
										#
										# - normal 25*8=~200MB (8 bytes per param)
										# - bnb    25*2= ~50MB (2 bytes per param)
										#
										# Thus we should expect ~150MB total memory saved.
										#
										# Peak memory should be the same - the total should be different by about that same margin
										#
										# After leaving a small margin to accommodate for differences between gpus let's check
										# that we have at least 120MB in savings
										__magic_name__					:       Dict    							=							120

										# uncomment the following if this test starts failing - requires py38 for a new print feature
										# gpu_peak_mem_diff = gpu_peak_mem_orig - gpu_peak_mem_bnb
										# print(f"{gpu_alloc_mem_orig=}MB {gpu_peak_mem_orig=}MB {gpu_alloc_mem_orig+gpu_peak_mem_orig=}MB")
										# print(f" {gpu_alloc_mem_bnb=}MB  {gpu_peak_mem_bnb=}MB  {gpu_alloc_mem_bnb+gpu_peak_mem_bnb=}MB")
										# print(f"{gpu_alloc_mem_diff=}MB")
										# print(f"{gpu_peak_mem_diff=}MB")
										# print(f"{gpu_total_mem_orig=}MB, {gpu_total_mem_bnb=}MB")
										# print(f"{gpu_total_mem_diff=}MB, {gpu_total_mem_diff=}MB")

										self.assertGreater(
										    lowerCamelCase__   ,				lowerCamelCase__   ,				"should use ~150MB less alloc gpu memory with BNB, compared to without it for this model but got"
										    f""" a difference of {gpu_alloc_mem_diff}MB, with gpu_alloc_mem_orig={gpu_alloc_mem_orig}MB and"""
										    f""" gpu_alloc_mem_bnb={gpu_alloc_mem_bnb}MB"""   ,				)

										self.assertGreater(
										    lowerCamelCase__   ,				lowerCamelCase__   ,				"should use ~150MB less total gpu memory with BNB, compared to without it for this model but got"
										    f""" a difference of {gpu_total_mem_diff}MB, with gpu_total_mem_orig={gpu_total_mem_orig}MB and"""
										    f""" gpu_total_mem_bnb={gpu_total_mem_bnb}MB"""   ,				)

										self.assertEqual(
										    lowerCamelCase__   ,				lowerCamelCase__   ,				f"""loss should be the same, but got loss_orig={loss_orig}, loss_bnb={loss_bnb}"""  )







						def lowerCAmelCase__			(	self:						int   ,				__UpperCamelCase:						int   ,				__UpperCamelCase:						str   ,				__UpperCamelCase:						int   ,				__UpperCamelCase:						float = 3E-3   ,				__UpperCamelCase:						str = "adafactor"   ,				__UpperCamelCase:						bool = False   ,				__UpperCamelCase:						str = None   ,				__UpperCamelCase:						int = 0   ,				__UpperCamelCase:						bool = True   ,				__UpperCamelCase:						bool = True   ,				__UpperCamelCase:						bool = True   ,				__UpperCamelCase:						bool = True   ,				__UpperCamelCase:						int = None   ,				)				->						List[str]:
										__magic_name__					:       Optional[Any]    							=							self.test_file_dir / "../fixtures/tests_samples/wmt_en_ro"
										__magic_name__					:       Dict    							=							self.get_auto_remove_tmp_dir()
										__magic_name__					:       Dict    							=							f"""\n            --model_name_or_path {model_name}\n            --train_file {data_dir}/train.json\n            --validation_file {data_dir}/val.json\n            --test_file {data_dir}/test.json\n            --output_dir {output_dir}\n            --overwrite_output_dir\n            --max_train_samples 8\n            --max_source_length {max_len}\n            --max_target_length {max_len}\n            --do_train\n            --num_train_epochs {str(lowerCamelCase__  )}\n            --per_device_train_batch_size 4\n            --learning_rate {learning_rate}\n            --warmup_steps 8\n            --logging_steps 0\n            --logging_strategy no\n            --save_steps {str(lowerCamelCase__  )}\n            --group_by_length\n            --label_smoothing_factor 0.1\n            --target_lang ro_RO\n            --source_lang en_XX\n        """.split()

										__magic_name__					:       Union[str, Any]    							=							f"""\n            --do_eval\n            --per_device_eval_batch_size 4\n            --max_eval_samples 8\n            --val_max_target_length {max_len}\n            --evaluation_strategy steps\n            --eval_steps {str(lowerCamelCase__  )}\n        """.split()

										__magic_name__					:       Any    							=							"\n            --do_predict\n        ".split()

										__magic_name__					:       Optional[int]    							=							[]
										if do_train:
														args += args_train

										if do_eval:
														args += args_eval

										if do_predict:
														args += args_predict

										if predict_with_generate:
														args += "--predict_with_generate".split()

										if do_train:
														if optim == "adafactor":
																		args += "--adafactor".split()
														else:
																		args += f"""--optim {optim}""".split()

										if extra_args_str is not None:
														args += extra_args_str.split()

										if distributed:
														if n_gpus_to_use is None:
																		__magic_name__					:       List[str]    							=							get_gpu_count()
														__magic_name__					:       str    							=							get_torch_dist_unique_port()
														__magic_name__					:       Optional[Any]    							=							f"""\n                -m torch.distributed.run\n                --nproc_per_node={n_gpus_to_use}\n                --master_port={master_port}\n                {self.examples_dir_str}/pytorch/translation/run_translation.py\n            """.split()
														__magic_name__					:       str    							=							[sys.executable] + distributed_args + args
														# keep for quick debug
														# print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
														execute_subprocess_async(lowerCamelCase__   ,				env=self.get_env()  )
										else:
														__magic_name__					:       int    							=							["run_translation.py"] + args
														with patch.object(lowerCamelCase__   ,				"argv"   ,				lowerCamelCase__  ):
																		main()

										return output_dir


import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
      __lowercase				      =						"""pt"""
elif is_tf_available():
      __lowercase				      =						"""tf"""
else:
      __lowercase				      =						"""jax"""

class    _lowercase				(					__lowerCamelCase,unittest.TestCase			):
   _lowercase     :       Dict  					=    PerceiverTokenizer
   _lowercase     :       Dict  					=    False

   def     UpperCamelCase					(				self	:    Tuple       ) ->			Any:
         """simple docstring"""





         super().setUp()
         A_							       =		PerceiverTokenizer()
         tokenizer.save_pretrained(self.tmpdirname       )

   @cached_property
   def     UpperCamelCase					(				self	:    Tuple       ) ->			List[str]:
         """simple docstring"""





         return PerceiverTokenizer.from_pretrained('''deepmind/language-perceiver'''       )

   def     UpperCamelCase					(				self	:    List[str]			,						**lowerCamelCase__	:    str       ) ->			PerceiverTokenizer:
         """simple docstring"""





         return self.tokenizer_class.from_pretrained(self.tmpdirname			,						**lowerCamelCase__       )

   def     UpperCamelCase					(				self	:    List[str]			,						lowerCamelCase__	:    str			,						lowerCamelCase__	:    Optional[Any]=False			,						lowerCamelCase__	:    List[str]=2_0			,						lowerCamelCase__	:    Union[str, Any]=5       ) ->			Tuple[str, list]:
         """simple docstring"""





         A_							       =		[]
         for i in range(len(lowerCamelCase__       )       ):
               try:
                     A_							       =		tokenizer.decode([i]			,						clean_up_tokenization_spaces=lowerCamelCase__       )
               except UnicodeDecodeError:
                     pass
               toks.append((i, tok)       )

         A_							       =		list(filter(lambda lowerCamelCase__       : re.match(R'''^[ a-zA-Z]+$'''			,						t[1]       )			,						lowerCamelCase__       )       )
         A_							       =		list(filter(lambda lowerCamelCase__       : [t[0]] == tokenizer.encode(t[1]			,						add_special_tokens=lowerCamelCase__       )			,						lowerCamelCase__       )       )
         if max_length is not None and len(lowerCamelCase__       ) > max_length:
               A_							       =		toks[:max_length]
         if min_length is not None and len(lowerCamelCase__       ) < min_length and len(lowerCamelCase__       ) > 0:
               while len(lowerCamelCase__       ) < min_length:
                     A_							       =		toks + toks
        # toks_str = [t[1] for t in toks]
         A_							       =		[t[0] for t in toks]

         # Ensure consistency
         A_							       =		tokenizer.decode(lowerCamelCase__			,						clean_up_tokenization_spaces=lowerCamelCase__       )
         if " " not in output_txt and len(lowerCamelCase__       ) > 1:
               A_							       =		(
                   tokenizer.decode([toks_ids[0]]			,						clean_up_tokenization_spaces=lowerCamelCase__       )
                   + ''' '''
                   + tokenizer.decode(toks_ids[1:]			,						clean_up_tokenization_spaces=lowerCamelCase__       )
               )
         if with_prefix_space:
               A_							       =		''' ''' + output_txt
         A_							       =		tokenizer.encode(lowerCamelCase__			,						add_special_tokens=lowerCamelCase__       )
         return output_txt, output_ids

   def     UpperCamelCase					(				self	:    Optional[Any]       ) ->			List[str]:
         """simple docstring"""





         A_							       =		self.perceiver_tokenizer
         A_							       =		'''Unicode €.'''
         A_							       =		tokenizer(lowerCamelCase__       )
         A_							       =		[4, 9_1, 1_1_6, 1_1_1, 1_0_5, 1_1_7, 1_0_6, 1_0_7, 3_8, 2_3_2, 1_3_6, 1_7_8, 5_2, 5]
         self.assertEqual(encoded['''input_ids''']			,						lowerCamelCase__       )

         # decoding
         A_							       =		tokenizer.decode(lowerCamelCase__       )
         self.assertEqual(lowerCamelCase__			,						'''[CLS]Unicode €.[SEP]'''       )

         A_							       =		tokenizer('''e è é ê ë'''       )
         A_							       =		[4, 1_0_7, 3_8, 2_0_1, 1_7_4, 3_8, 2_0_1, 1_7_5, 3_8, 2_0_1, 1_7_6, 3_8, 2_0_1, 1_7_7, 5]
         self.assertEqual(encoded['''input_ids''']			,						lowerCamelCase__       )
         # decoding
         A_							       =		tokenizer.decode(lowerCamelCase__       )
         self.assertEqual(lowerCamelCase__			,						'''[CLS]e è é ê ë[SEP]'''       )

         # encode/decode, but with `encode` instead of `__call__`
         self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''       )       )			,						'''[CLS]e è é ê ë[SEP]'''       )

   def     UpperCamelCase					(				self	:    Union[str, Any]       ) ->			Union[str, Any]:
         """simple docstring"""





         A_							       =		self.perceiver_tokenizer
         A_							       =		['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
         # fmt: off
         A_							       =		[4, 7_1, 3_8, 1_1_4, 1_1_7, 1_1_6, 1_0_9, 3_8, 1_1_8, 1_0_3, 1_2_0, 1_0_3, 1_0_9, 1_2_0, 1_0_3, 1_1_8, 1_1_0, 3_8, 1_0_8, 1_1_7, 1_2_0, 3_8, 1_2_1, 1_2_3, 1_1_5, 1_1_5, 1_0_3, 1_2_0, 1_1_1, 1_2_8, 1_0_3, 1_2_2, 1_1_1, 1_1_7, 1_1_6, 5_2, 5, 0]
         # fmt: on
         A_							       =		tokenizer(lowerCamelCase__			,						padding=lowerCamelCase__			,						return_tensors=lowerCamelCase__       )
         self.assertIsInstance(lowerCamelCase__			,						lowerCamelCase__       )

         if FRAMEWORK != "jax":
               A_							       =		list(batch.input_ids.numpy()[0]       )
         else:
               A_							       =		list(batch.input_ids.tolist()[0]       )

         self.assertListEqual(lowerCamelCase__			,						lowerCamelCase__       )

         self.assertEqual((2, 3_8)			,						batch.input_ids.shape       )
         self.assertEqual((2, 3_8)			,						batch.attention_mask.shape       )

   def     UpperCamelCase					(				self	:    str       ) ->			List[str]:
         """simple docstring"""





         A_							       =		self.perceiver_tokenizer
         A_							       =		['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
         A_							       =		tokenizer(lowerCamelCase__			,						padding=lowerCamelCase__			,						return_tensors=lowerCamelCase__       )
         # check if input_ids are returned and no decoder_input_ids
         self.assertIn('''input_ids'''			,						lowerCamelCase__       )
         self.assertIn('''attention_mask'''			,						lowerCamelCase__       )
         self.assertNotIn('''decoder_input_ids'''			,						lowerCamelCase__       )
         self.assertNotIn('''decoder_attention_mask'''			,						lowerCamelCase__       )

   def     UpperCamelCase					(				self	:    Any       ) ->			Union[str, Any]:
         """simple docstring"""





         A_							       =		self.perceiver_tokenizer
         A_							       =		[
             '''Summary of the text.''',
             '''Another summary.''',
         ]
         A_							       =		tokenizer(
             text_target=lowerCamelCase__			,						max_length=3_2			,						padding='''max_length'''			,						truncation=lowerCamelCase__			,						return_tensors=lowerCamelCase__       )
         self.assertEqual(3_2			,						targets['''input_ids'''].shape[1]       )

   def     UpperCamelCase					(				self	:    Dict       ) ->			List[str]:
         """simple docstring"""





         A_							       =		self.get_tokenizers()
         for tokenizer in tokenizers:
               with self.subTest(F"{tokenizer.__class__.__name__}"       ):
                     self.assertNotEqual(tokenizer.model_max_length			,						4_2       )

        # Now let's start the test
         A_							       =		self.get_tokenizers()
         for tokenizer in tokenizers:
               with self.subTest(F"{tokenizer.__class__.__name__}"       ):
                     # Isolate this from the other tests because we save additional tokens/etc
                     A_							       =		tempfile.mkdtemp()

                     A_							       =		''' He is very happy, UNwant\u00E9d,running'''
                     A_							       =		tokenizer.encode(lowerCamelCase__			,						add_special_tokens=lowerCamelCase__       )
                     tokenizer.save_pretrained(lowerCamelCase__       )

                     A_							       =		tokenizer.__class__.from_pretrained(lowerCamelCase__       )
                     A_							       =		after_tokenizer.encode(lowerCamelCase__			,						add_special_tokens=lowerCamelCase__       )
                     self.assertListEqual(lowerCamelCase__			,						lowerCamelCase__       )

                     shutil.rmtree(lowerCamelCase__       )

         A_							       =		self.get_tokenizers(model_max_length=4_2       )
         for tokenizer in tokenizers:
               with self.subTest(F"{tokenizer.__class__.__name__}"       ):
                     # Isolate this from the other tests because we save additional tokens/etc
                     A_							       =		tempfile.mkdtemp()

                     A_							       =		''' He is very happy, UNwant\u00E9d,running'''
                     tokenizer.add_tokens(['''bim''', '''bambam''']       )
                     A_							       =		tokenizer.additional_special_tokens
                     additional_special_tokens.append('''new_additional_special_token'''       )
                     tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}       )
                     A_							       =		tokenizer.encode(lowerCamelCase__			,						add_special_tokens=lowerCamelCase__       )
                     tokenizer.save_pretrained(lowerCamelCase__       )

                     A_							       =		tokenizer.__class__.from_pretrained(lowerCamelCase__       )
                     A_							       =		after_tokenizer.encode(lowerCamelCase__			,						add_special_tokens=lowerCamelCase__       )
                     self.assertListEqual(lowerCamelCase__			,						lowerCamelCase__       )
                     self.assertIn('''new_additional_special_token'''			,						after_tokenizer.additional_special_tokens       )
                     self.assertEqual(after_tokenizer.model_max_length			,						4_2       )

                     A_							       =		tokenizer.__class__.from_pretrained(lowerCamelCase__			,						model_max_length=4_3       )
                     self.assertEqual(tokenizer.model_max_length			,						4_3       )

                     shutil.rmtree(lowerCamelCase__       )

   def     UpperCamelCase					(				self	:    List[Any]       ) ->			Any:
         """simple docstring"""





         A_							       =		[]
         if self.test_slow_tokenizer:
               tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

         if self.test_rust_tokenizer:
               tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

         for tokenizer_class, tokenizer_utils in tokenizer_list:
               with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer_utils.save_pretrained(lowerCamelCase__       )

                     with open(os.path.join(lowerCamelCase__			,						'''special_tokens_map.json'''       )			,						encoding='''utf-8'''       ) as json_file:
                           A_							       =		json.load(lowerCamelCase__       )

                     with open(os.path.join(lowerCamelCase__			,						'''tokenizer_config.json'''       )			,						encoding='''utf-8'''       ) as json_file:
                           A_							       =		json.load(lowerCamelCase__       )

                     A_							       =		[F"<extra_id_{i}>" for i in range(1_2_5       )]

                     A_							       =		added_tokens_extra_ids + [
                         '''an_additional_special_token'''
                     ]
                     A_							       =		added_tokens_extra_ids + [
                         '''an_additional_special_token'''
                     ]

                     with open(os.path.join(lowerCamelCase__			,						'''special_tokens_map.json'''       )			,						'''w'''			,						encoding='''utf-8'''       ) as outfile:
                           json.dump(lowerCamelCase__			,						lowerCamelCase__       )
                     with open(os.path.join(lowerCamelCase__			,						'''tokenizer_config.json'''       )			,						'''w'''			,						encoding='''utf-8'''       ) as outfile:
                           json.dump(lowerCamelCase__			,						lowerCamelCase__       )

                     # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                     # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                     # "special_tokens_map.json" files
                     A_							       =		tokenizer_class.from_pretrained(
                         lowerCamelCase__			,						)
                     self.assertIn(
                         '''an_additional_special_token'''			,						tokenizer_without_change_in_init.additional_special_tokens       )
                     self.assertEqual(
                         ['''an_additional_special_token''']			,						tokenizer_without_change_in_init.convert_ids_to_tokens(
                             tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']       )       )			,						)

                     # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                     A_							       =		added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''			,						lstrip=lowerCamelCase__       )]
                     A_							       =		tokenizer_class.from_pretrained(
                         lowerCamelCase__			,						additional_special_tokens=lowerCamelCase__			,						)

                     self.assertIn('''a_new_additional_special_token'''			,						tokenizer.additional_special_tokens       )
                     self.assertEqual(
                         ['''a_new_additional_special_token''']			,						tokenizer.convert_ids_to_tokens(
                             tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']       )       )			,						)

   def     UpperCamelCase					(				self	:    Dict       ) ->			List[Any]:
         """simple docstring"""





         A_							       =		self.perceiver_tokenizer
         self.assertEqual(tokenizer.decode([1_7_8]       )			,						'''�'''       )

   def     UpperCamelCase					(				self	:    int       ) ->			Any:
         """simple docstring"""





         pass

   def     UpperCamelCase					(				self	:    int       ) ->			int:
         """simple docstring"""





         pass

   def     UpperCamelCase					(				self	:    List[str]       ) ->			Dict:
         """simple docstring"""





         pass

   def     UpperCamelCase					(				self	:    int       ) ->			Any:
         """simple docstring"""





         pass





   def     UpperCamelCase					(				self	:    List[Any]       ) ->			Dict:
         """simple docstring"""





         A_							       =		self.get_tokenizers(fast=lowerCamelCase__			,						do_lower_case=lowerCamelCase__       )
         for tokenizer in tokenizers:
               with self.subTest(F"{tokenizer.__class__.__name__}"       ):
                     A_							       =		['''[CLS]''', '''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''s''', '''t''', '''[SEP]''']
                     A_							       =		tokenizer.convert_tokens_to_string(lowerCamelCase__       )

                     self.assertIsInstance(lowerCamelCase__			,						lowerCamelCase__       )





