"""simple docstring"""




import json
import pathlib
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision, slow
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
				import torch

if is_vision_available():
				from PIL import Image

				from transformers import DetrImageProcessor
class     UpperCamelCase__(			unittest.TestCase ):





						def __init__(       self		,__UpperCAmelCase		,__UpperCAmelCase=7		,__UpperCAmelCase=3		,__UpperCAmelCase=30		,__UpperCAmelCase=4_00		,__UpperCAmelCase=True		,__UpperCAmelCase=None		,__UpperCAmelCase=True		,__UpperCAmelCase=1 / 2_55		,__UpperCAmelCase=True		,__UpperCAmelCase=[0.5, 0.5, 0.5]		,__UpperCAmelCase=[0.5, 0.5, 0.5]		,__UpperCAmelCase=True		,)					->				Optional[Any]:
													# by setting size["longest_edge"] > max_resolution we're effectively not testing this :p
													A__	     =							size if size is not None else {'shortest_edge': 18, 'longest_edge': 13_33}
													A__	     =							parent
													A__	     =							batch_size
													A__	     =							num_channels
													A__	     =							min_resolution
													A__	     =							max_resolution
													A__	     =							do_resize
													A__	     =							size
													A__	     =							do_rescale
													A__	     =							rescale_factor
													A__	     =							do_normalize
													A__	     =							image_mean
													A__	     =							image_std
													A__	     =							do_pad





						def 				snake_case__	(       self     )					->				Dict:
													return {
													    "do_resize": self.do_resize,
													    "size": self.size,
													    "do_rescale": self.do_rescale,
													    "rescale_factor": self.rescale_factor,
													    "do_normalize": self.do_normalize,
													    "image_mean": self.image_mean,
													    "image_std": self.image_std,
													    "do_pad": self.do_pad,
													}





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase=False     )					->				Dict:
													if not batched:
																				A__	     =							image_inputs[0]
																				if isinstance(__UpperCAmelCase		,Image.Image     ):
																											A__     ,  A__	     =							image.size
																				else:
																											A__     ,  A__	     =							image.shape[1], image.shape[2]
																				if w < h:
																											A__	     =							int(self.size['shortest_edge'] * h / w     )
																											A__	     =							self.size['shortest_edge']
																				elif w > h:
																											A__	     =							self.size['shortest_edge']
																											A__	     =							int(self.size['shortest_edge'] * w / h     )
																				else:
																											A__	     =							self.size['shortest_edge']
																											A__	     =							self.size['shortest_edge']

													else:
																				A__	     =							[]
																				for image in image_inputs:
																											A__     ,  A__	     =							self.get_expected_values([image]     )
																											expected_values.append((expected_height, expected_width)     )
																				A__	     =							max(__UpperCAmelCase		,key=lambda __UpperCAmelCase     : item[0]     )[0]
																				A__	     =							max(__UpperCAmelCase		,key=lambda __UpperCAmelCase     : item[1]     )[1]

													return expected_height, expected_width






@require_torch
@require_vision
class     UpperCamelCase__(			__A		,						unittest.TestCase ):
						lowerCAmelCase__  :						Any													=		DetrImageProcessor if is_vision_available() else None





						def 				snake_case__	(       self     )					->				int:
													A__	     =							DetrImageProcessingTester(self     )





						@property
						def 				snake_case__	(       self     )					->				List[Any]:
													return self.image_processor_tester.prepare_image_processor_dict()





						def 				snake_case__	(       self     )					->				Tuple:
													A__	     =							self.image_processing_class(**self.image_processor_dict     )
													self.assertTrue(hasattr(__UpperCAmelCase		,'image_mean'     )     )
													self.assertTrue(hasattr(__UpperCAmelCase		,'image_std'     )     )
													self.assertTrue(hasattr(__UpperCAmelCase		,'do_normalize'     )     )
													self.assertTrue(hasattr(__UpperCAmelCase		,'do_rescale'     )     )
													self.assertTrue(hasattr(__UpperCAmelCase		,'rescale_factor'     )     )
													self.assertTrue(hasattr(__UpperCAmelCase		,'do_resize'     )     )
													self.assertTrue(hasattr(__UpperCAmelCase		,'size'     )     )
													self.assertTrue(hasattr(__UpperCAmelCase		,'do_pad'     )     )





						def 				snake_case__	(       self     )					->				Optional[int]:
													A__	     =							self.image_processing_class.from_dict(self.image_processor_dict     )
													self.assertEqual(image_processor.size		,{'shortest_edge': 18, 'longest_edge': 13_33}     )
													self.assertEqual(image_processor.do_pad		,__UpperCAmelCase     )

													A__	     =							self.image_processing_class.from_dict(
													    self.image_processor_dict		,size=42		,max_size=84		,pad_and_return_pixel_mask=__UpperCAmelCase     )
													self.assertEqual(image_processor.size		,{'shortest_edge': 42, 'longest_edge': 84}     )
													self.assertEqual(image_processor.do_pad		,__UpperCAmelCase     )





						def 				snake_case__	(       self     )					->				Any:
													pass





						def 				snake_case__	(       self     )					->				Dict:
													# Initialize image_processing
													A__	     =							self.image_processing_class(**self.image_processor_dict     )
													# create random PIL images
													A__	     =							prepare_image_inputs(self.image_processor_tester		,equal_resolution=__UpperCAmelCase     )
													for image in image_inputs:
																				self.assertIsInstance(__UpperCAmelCase		,Image.Image     )

													# Test not batched input
													A__	     =							image_processing(image_inputs[0]		,return_tensors='pt'     ).pixel_values

													A__     ,  A__	     =							self.image_processor_tester.get_expected_values(__UpperCAmelCase     )

													self.assertEqual(
													    encoded_images.shape		,(1, self.image_processor_tester.num_channels, expected_height, expected_width)		,)

													# Test batched
													A__     ,  A__	     =							self.image_processor_tester.get_expected_values(__UpperCAmelCase		,batched=__UpperCAmelCase     )

													A__	     =							image_processing(__UpperCAmelCase		,return_tensors='pt'     ).pixel_values
													self.assertEqual(
													    encoded_images.shape		,(
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        expected_height,
													        expected_width,
													    )		,)





						def 				snake_case__	(       self     )					->				List[str]:
													# Initialize image_processing
													A__	     =							self.image_processing_class(**self.image_processor_dict     )
													# create random numpy tensors
													A__	     =							prepare_image_inputs(self.image_processor_tester		,equal_resolution=__UpperCAmelCase		,numpify=__UpperCAmelCase     )
													for image in image_inputs:
																				self.assertIsInstance(__UpperCAmelCase		,np.ndarray     )

													# Test not batched input
													A__	     =							image_processing(image_inputs[0]		,return_tensors='pt'     ).pixel_values

													A__     ,  A__	     =							self.image_processor_tester.get_expected_values(__UpperCAmelCase     )

													self.assertEqual(
													    encoded_images.shape		,(1, self.image_processor_tester.num_channels, expected_height, expected_width)		,)

													# Test batched
													A__	     =							image_processing(__UpperCAmelCase		,return_tensors='pt'     ).pixel_values

													A__     ,  A__	     =							self.image_processor_tester.get_expected_values(__UpperCAmelCase		,batched=__UpperCAmelCase     )

													self.assertEqual(
													    encoded_images.shape		,(
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        expected_height,
													        expected_width,
													    )		,)





						def 				snake_case__	(       self     )					->				Dict:
													# Initialize image_processing
													A__	     =							self.image_processing_class(**self.image_processor_dict     )
													# create random PyTorch tensors
													A__	     =							prepare_image_inputs(self.image_processor_tester		,equal_resolution=__UpperCAmelCase		,torchify=__UpperCAmelCase     )
													for image in image_inputs:
																				self.assertIsInstance(__UpperCAmelCase		,torch.Tensor     )

													# Test not batched input
													A__	     =							image_processing(image_inputs[0]		,return_tensors='pt'     ).pixel_values

													A__     ,  A__	     =							self.image_processor_tester.get_expected_values(__UpperCAmelCase     )

													self.assertEqual(
													    encoded_images.shape		,(1, self.image_processor_tester.num_channels, expected_height, expected_width)		,)

													# Test batched
													A__	     =							image_processing(__UpperCAmelCase		,return_tensors='pt'     ).pixel_values

													A__     ,  A__	     =							self.image_processor_tester.get_expected_values(__UpperCAmelCase		,batched=__UpperCAmelCase     )

													self.assertEqual(
													    encoded_images.shape		,(
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        expected_height,
													        expected_width,
													    )		,)





						@slow
						def 				snake_case__	(       self     )					->				Tuple:
													# prepare image and target
													A__	     =							Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'     )
													with open('./tests/fixtures/tests_samples/COCO/coco_annotations.txt'		,'r'     ) as f:
																				A__	     =							json.loads(f.read()     )

													A__	     =							{'image_id': 3_97_69, 'annotations': target}

													# encode them
													A__	     =							DetrImageProcessor.from_pretrained('facebook/detr-resnet-50'     )
													A__	     =							image_processing(images=__UpperCAmelCase		,annotations=__UpperCAmelCase		,return_tensors='pt'     )

													# verify pixel values
													A__	     =							torch.Size([1, 3, 8_00, 10_66]     )
													self.assertEqual(encoding['pixel_values'].shape		,__UpperCAmelCase     )

													A__	     =							torch.tensor([0.2_7_9_6, 0.3_1_3_8, 0.3_4_8_1]     )
													self.assertTrue(torch.allclose(encoding['pixel_values'][0, 0, 0, :3]		,__UpperCAmelCase		,atol=1e-4     )     )

													# verify area
													A__	     =							torch.tensor([5_8_8_7.9_6_0_0, 1_1_2_5_0.2_0_6_1, 4_8_9_3_5_3.8_4_3_8, 8_3_7_1_2_2.7_5_0_0, 1_4_7_9_6_7.5_1_5_6, 1_6_5_7_3_2.3_4_3_8]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['area']		,__UpperCAmelCase     )     )
													# verify boxes
													A__	     =							torch.Size([6, 4]     )
													self.assertEqual(encoding['labels'][0]['boxes'].shape		,__UpperCAmelCase     )
													A__	     =							torch.tensor([0.5_5_0_3, 0.2_7_6_5, 0.0_6_0_4, 0.2_2_1_5]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['boxes'][0]		,__UpperCAmelCase		,atol=1e-3     )     )
													# verify image_id
													A__	     =							torch.tensor([3_97_69]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['image_id']		,__UpperCAmelCase     )     )
													# verify is_crowd
													A__	     =							torch.tensor([0, 0, 0, 0, 0, 0]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['iscrowd']		,__UpperCAmelCase     )     )
													# verify class_labels
													A__	     =							torch.tensor([75, 75, 63, 65, 17, 17]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['class_labels']		,__UpperCAmelCase     )     )
													# verify orig_size
													A__	     =							torch.tensor([4_80, 6_40]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['orig_size']		,__UpperCAmelCase     )     )
													# verify size
													A__	     =							torch.tensor([8_00, 10_66]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['size']		,__UpperCAmelCase     )     )





						@slow
						def 				snake_case__	(       self     )					->				Any:
													# prepare image, target and masks_path
													A__	     =							Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'     )
													with open('./tests/fixtures/tests_samples/COCO/coco_panoptic_annotations.txt'		,'r'     ) as f:
																				A__	     =							json.loads(f.read()     )

													A__	     =							{'file_name': '000000039769.png', 'image_id': 3_97_69, 'segments_info': target}

													A__	     =							pathlib.Path('./tests/fixtures/tests_samples/COCO/coco_panoptic'     )

													# encode them
													A__	     =							DetrImageProcessor.from_pretrained('facebook/detr-resnet-50-panoptic'     )
													A__	     =							image_processing(images=__UpperCAmelCase		,annotations=__UpperCAmelCase		,masks_path=__UpperCAmelCase		,return_tensors='pt'     )

													# verify pixel values
													A__	     =							torch.Size([1, 3, 8_00, 10_66]     )
													self.assertEqual(encoding['pixel_values'].shape		,__UpperCAmelCase     )

													A__	     =							torch.tensor([0.2_7_9_6, 0.3_1_3_8, 0.3_4_8_1]     )
													self.assertTrue(torch.allclose(encoding['pixel_values'][0, 0, 0, :3]		,__UpperCAmelCase		,atol=1e-4     )     )

													# verify area
													A__	     =							torch.tensor([1_4_7_9_7_9.6_8_7_5, 1_6_5_5_2_7.0_4_6_9, 4_8_4_6_3_8.5_9_3_8, 1_1_2_9_2.9_3_7_5, 5_8_7_9.6_5_6_2, 7_6_3_4.1_1_4_7]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['area']		,__UpperCAmelCase     )     )
													# verify boxes
													A__	     =							torch.Size([6, 4]     )
													self.assertEqual(encoding['labels'][0]['boxes'].shape		,__UpperCAmelCase     )
													A__	     =							torch.tensor([0.2_6_2_5, 0.5_4_3_7, 0.4_6_8_8, 0.8_6_2_5]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['boxes'][0]		,__UpperCAmelCase		,atol=1e-3     )     )
													# verify image_id
													A__	     =							torch.tensor([3_97_69]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['image_id']		,__UpperCAmelCase     )     )
													# verify is_crowd
													A__	     =							torch.tensor([0, 0, 0, 0, 0, 0]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['iscrowd']		,__UpperCAmelCase     )     )
													# verify class_labels
													A__	     =							torch.tensor([17, 17, 63, 75, 75, 93]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['class_labels']		,__UpperCAmelCase     )     )
													# verify masks
													A__	     =							82_28_73
													self.assertEqual(encoding['labels'][0]['masks'].sum().item()		,__UpperCAmelCase     )
													# verify orig_size
													A__	     =							torch.tensor([4_80, 6_40]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['orig_size']		,__UpperCAmelCase     )     )
													# verify size
													A__	     =							torch.tensor([8_00, 10_66]     )
													self.assertTrue(torch.allclose(encoding['labels'][0]['size']		,__UpperCAmelCase     )     )






"""simple docstring"""




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
				__lowerCamelCase											=  "pt"
elif is_tf_available():
				__lowerCamelCase											=  "tf"
else:
				__lowerCamelCase											=  "jax"
class     UpperCamelCase__(			__A		,						unittest.TestCase ):
						lowerCAmelCase__  :						List[Any]													=		ByTaTokenizer
						lowerCAmelCase__  :						str													=		False





						def 				snake_case__	(       self     )					->				int:
													super().setUp()
													A__	     =							ByTaTokenizer()
													tokenizer.save_pretrained(self.tmpdirname     )





						@cached_property
						def 				snake_case__	(       self     )					->				List[Any]:
													return ByTaTokenizer.from_pretrained('google/byt5-small'     )





						def 				snake_case__	(       self		,**__UpperCAmelCase     )					->				ByTaTokenizer:
													return self.tokenizer_class.from_pretrained(self.tmpdirname		,**__UpperCAmelCase     )





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase=False		,__UpperCAmelCase=20		,__UpperCAmelCase=5     )					->				Tuple[str, list]:
													# XXX The default common tokenizer tests assume that every ID is decodable on its own.
													# This assumption is invalid for ByT5 because single bytes might not be
													# valid utf-8 (byte 128 for instance).
													# Here we're overriding the smallest possible method to provide
													# a clean sequence without making the same assumption.

													A__	     =							[]
													for i in range(len(__UpperCAmelCase     )     ):
																				try:
																											A__	     =							tokenizer.decode([i]		,clean_up_tokenization_spaces=__UpperCAmelCase     )
																				except UnicodeDecodeError:
																											pass
																				toks.append((i, tok)     )

													A__	     =							list(filter(lambda __UpperCAmelCase     : re.match(R'^[ a-zA-Z]+$'		,t[1]     )		,__UpperCAmelCase     )     )
													A__	     =							list(filter(lambda __UpperCAmelCase     : [t[0]] == tokenizer.encode(t[1]		,add_special_tokens=__UpperCAmelCase     )		,__UpperCAmelCase     )     )
													if max_length is not None and len(__UpperCAmelCase     ) > max_length:
																				A__	     =							toks[:max_length]
													if min_length is not None and len(__UpperCAmelCase     ) < min_length and len(__UpperCAmelCase     ) > 0:
																				while len(__UpperCAmelCase     ) < min_length:
																											A__	     =							toks + toks
        # toks_str = [t[1] for t in toks]
													A__	     =							[t[0] for t in toks]

													# Ensure consistency
													A__	     =							tokenizer.decode(__UpperCAmelCase		,clean_up_tokenization_spaces=__UpperCAmelCase     )
													if " " not in output_txt and len(__UpperCAmelCase     ) > 1:
																				A__	     =							(
																				    tokenizer.decode([toks_ids[0]]		,clean_up_tokenization_spaces=__UpperCAmelCase     )
																				    + ' '
																				    + tokenizer.decode(toks_ids[1:]		,clean_up_tokenization_spaces=__UpperCAmelCase     )
																				)
													if with_prefix_space:
																				A__	     =							' ' + output_txt
													A__	     =							tokenizer.encode(__UpperCAmelCase		,add_special_tokens=__UpperCAmelCase     )
													return output_txt, output_ids





						def 				snake_case__	(       self     )					->				Dict:
													A__	     =							self.ta_base_tokenizer
													A__	     =							tokenizer(['hi</s>', 'I went to the gym</s>', '</s>']     )
													A__	     =							tokenizer(['hi', 'I went to the gym', '']     )
													self.assertListEqual(batch_with_eos_added['input_ids']		,batch_without_eos_added['input_ids']     )





						def 				snake_case__	(       self     )					->				Dict:
													A__	     =							self.ta_base_tokenizer
													A__	     =							'Unicode €.'
													A__	     =							tokenizer(__UpperCAmelCase     )
													A__	     =							[88, 1_13, 1_08, 1_02, 1_14, 1_03, 1_04, 35, 2_29, 1_33, 1_75, 49, 1]
													self.assertEqual(encoded['input_ids']		,__UpperCAmelCase     )

													# decoding
													A__	     =							tokenizer.decode(__UpperCAmelCase     )
													self.assertEqual(__UpperCAmelCase		,'Unicode €.</s>'     )

													A__	     =							tokenizer('e è é ê ë'     )
													A__	     =							[1_04, 35, 1_98, 1_71, 35, 1_98, 1_72, 35, 1_98, 1_73, 35, 1_98, 1_74, 1]
													self.assertEqual(encoded['input_ids']		,__UpperCAmelCase     )
													# decoding
													A__	     =							tokenizer.decode(__UpperCAmelCase     )
													self.assertEqual(__UpperCAmelCase		,'e è é ê ë</s>'     )

													# encode/decode, but with `encode` instead of `__call__`
													self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'     )     )		,'e è é ê ë</s>'     )





						def 				snake_case__	(       self     )					->				Optional[int]:
													A__	     =							self.ta_base_tokenizer
													A__	     =							['A long paragraph for summarization.', 'Another paragraph for summarization.']
													# fmt: off
													A__	     =							[68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 1, 0]
													# fmt: on
													A__	     =							tokenizer(__UpperCAmelCase		,padding=__UpperCAmelCase		,return_tensors=__UpperCAmelCase     )
													self.assertIsInstance(__UpperCAmelCase		,__UpperCAmelCase     )

													if FRAMEWORK != "jax":
																				A__	     =							list(batch.input_ids.numpy()[0]     )
													else:
																				A__	     =							list(batch.input_ids.tolist()[0]     )

													self.assertListEqual(__UpperCAmelCase		,__UpperCAmelCase     )

													self.assertEqual((2, 37)		,batch.input_ids.shape     )
													self.assertEqual((2, 37)		,batch.attention_mask.shape     )





						def 				snake_case__	(       self     )					->				List[str]:
													A__	     =							self.ta_base_tokenizer
													A__	     =							['A long paragraph for summarization.', 'Another paragraph for summarization.']
													A__	     =							tokenizer(__UpperCAmelCase		,padding=__UpperCAmelCase		,return_tensors=__UpperCAmelCase     )
													# check if input_ids are returned and no decoder_input_ids
													self.assertIn('input_ids'		,__UpperCAmelCase     )
													self.assertIn('attention_mask'		,__UpperCAmelCase     )
													self.assertNotIn('decoder_input_ids'		,__UpperCAmelCase     )
													self.assertNotIn('decoder_attention_mask'		,__UpperCAmelCase     )





						def 				snake_case__	(       self     )					->				Any:
													A__	     =							self.ta_base_tokenizer
													A__	     =							[
													    'Summary of the text.',
													    'Another summary.',
													]
													A__	     =							tokenizer(
													    text_target=__UpperCAmelCase		,max_length=32		,padding='max_length'		,truncation=__UpperCAmelCase		,return_tensors=__UpperCAmelCase     )
													self.assertEqual(32		,targets['input_ids'].shape[1]     )





						def 				snake_case__	(       self     )					->				Optional[int]:
													A__	     =							self.ta_base_tokenizer
													A__	     =							['A long paragraph for summarization. </s>']
													A__	     =							['Summary of the text. </s>']
													# fmt: off
													A__	     =							[68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 35, 1]
													A__	     =							[86, 1_20, 1_12, 1_12, 1_00, 1_17, 1_24, 35, 1_14, 1_05, 35, 1_19, 1_07, 1_04, 35, 1_19, 1_04, 1_23, 1_19, 49, 35, 1]
													# fmt: on

													A__	     =							tokenizer(__UpperCAmelCase		,text_target=__UpperCAmelCase     )

													self.assertEqual(__UpperCAmelCase		,batch['input_ids'][0]     )
													self.assertEqual(__UpperCAmelCase		,batch['labels'][0]     )





						def 				snake_case__	(       self     )					->				Tuple:
													# safety check on max_len default value so we are sure the test works
													A__	     =							self.get_tokenizers()
													for tokenizer in tokenizers:
																				with self.subTest(f'''{tokenizer.__class__.__name__}'''     ):
																											self.assertNotEqual(tokenizer.model_max_length		,42     )

        # Now let's start the test
													A__	     =							self.get_tokenizers()
													for tokenizer in tokenizers:
																				with self.subTest(f'''{tokenizer.__class__.__name__}'''     ):
																											# Isolate this from the other tests because we save additional tokens/etc
																											A__	     =							tempfile.mkdtemp()

																											A__	     =							' He is very happy, UNwant\u00E9d,running'
																											A__	     =							tokenizer.encode(__UpperCAmelCase		,add_special_tokens=__UpperCAmelCase     )
																											tokenizer.save_pretrained(__UpperCAmelCase     )

																											A__	     =							tokenizer.__class__.from_pretrained(__UpperCAmelCase     )
																											A__	     =							after_tokenizer.encode(__UpperCAmelCase		,add_special_tokens=__UpperCAmelCase     )
																											self.assertListEqual(__UpperCAmelCase		,__UpperCAmelCase     )

																											shutil.rmtree(__UpperCAmelCase     )

													A__	     =							self.get_tokenizers(model_max_length=42     )
													for tokenizer in tokenizers:
																				with self.subTest(f'''{tokenizer.__class__.__name__}'''     ):
																											# Isolate this from the other tests because we save additional tokens/etc
																											A__	     =							tempfile.mkdtemp()

																											A__	     =							' He is very happy, UNwant\u00E9d,running'
																											tokenizer.add_tokens(['bim', 'bambam']     )
																											A__	     =							tokenizer.additional_special_tokens
																											additional_special_tokens.append('new_additional_special_token'     )
																											tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}     )
																											A__	     =							tokenizer.encode(__UpperCAmelCase		,add_special_tokens=__UpperCAmelCase     )
																											tokenizer.save_pretrained(__UpperCAmelCase     )

																											A__	     =							tokenizer.__class__.from_pretrained(__UpperCAmelCase     )
																											A__	     =							after_tokenizer.encode(__UpperCAmelCase		,add_special_tokens=__UpperCAmelCase     )
																											self.assertListEqual(__UpperCAmelCase		,__UpperCAmelCase     )
																											self.assertIn('new_additional_special_token'		,after_tokenizer.additional_special_tokens     )
																											self.assertEqual(after_tokenizer.model_max_length		,42     )

																											A__	     =							tokenizer.__class__.from_pretrained(__UpperCAmelCase		,model_max_length=43     )
																											self.assertEqual(tokenizer.model_max_length		,43     )

																											shutil.rmtree(__UpperCAmelCase     )





						def 				snake_case__	(       self     )					->				Optional[int]:
													A__	     =							[]
													if self.test_slow_tokenizer:
																				tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

													if self.test_rust_tokenizer:
																				tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

													for tokenizer_class, tokenizer_utils in tokenizer_list:
																				with tempfile.TemporaryDirectory() as tmp_dir:
																											tokenizer_utils.save_pretrained(__UpperCAmelCase     )

																											with open(os.path.join(__UpperCAmelCase		,'special_tokens_map.json'     )		,encoding='utf-8'     ) as json_file:
																																		A__	     =							json.load(__UpperCAmelCase     )

																											with open(os.path.join(__UpperCAmelCase		,'tokenizer_config.json'     )		,encoding='utf-8'     ) as json_file:
																																		A__	     =							json.load(__UpperCAmelCase     )

																											A__	     =							[f'''<extra_id_{i}>''' for i in range(1_25     )]

																											A__	     =							added_tokens_extra_ids + [
																											    'an_additional_special_token'
																											]
																											A__	     =							added_tokens_extra_ids + [
																											    'an_additional_special_token'
																											]

																											with open(os.path.join(__UpperCAmelCase		,'special_tokens_map.json'     )		,'w'		,encoding='utf-8'     ) as outfile:
																																		json.dump(__UpperCAmelCase		,__UpperCAmelCase     )
																											with open(os.path.join(__UpperCAmelCase		,'tokenizer_config.json'     )		,'w'		,encoding='utf-8'     ) as outfile:
																																		json.dump(__UpperCAmelCase		,__UpperCAmelCase     )

																											# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																											# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																											# "special_tokens_map.json" files
																											A__	     =							tokenizer_class.from_pretrained(
																											    __UpperCAmelCase		,)
																											self.assertIn(
																											    'an_additional_special_token'		,tokenizer_without_change_in_init.additional_special_tokens     )
																											# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																											self.assertEqual(
																											    ['an_additional_special_token']		,tokenizer_without_change_in_init.convert_ids_to_tokens(
																											        tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']     )     )		,)

																											# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																											A__	     =							added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'		,lstrip=__UpperCAmelCase     )]
																											A__	     =							tokenizer_class.from_pretrained(
																											    __UpperCAmelCase		,additional_special_tokens=__UpperCAmelCase		,)

																											self.assertIn('a_new_additional_special_token'		,tokenizer.additional_special_tokens     )
																											self.assertEqual(
																											    ['a_new_additional_special_token']		,tokenizer.convert_ids_to_tokens(
																											        tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']     )     )		,)





						def 				snake_case__	(       self     )					->				str:
													A__	     =							[]
													if self.test_slow_tokenizer:
																				tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

													if self.test_rust_tokenizer:
																				tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

													for tokenizer_class, tokenizer_utils in tokenizer_list:
																				with tempfile.TemporaryDirectory() as tmp_dir:
																											tokenizer_utils.save_pretrained(__UpperCAmelCase     )

																											A__	     =							tokenizer_class.from_pretrained(__UpperCAmelCase     )

																											self.assertTrue(tokenizer.decode([2_55]     ) == ''     )





						def 				snake_case__	(       self     )					->				Any:
													pass





						def 				snake_case__	(       self     )					->				Any:
													pass





						def 				snake_case__	(       self     )					->				str:
													pass





						def 				snake_case__	(       self     )					->				int:
													pass





						def 				snake_case__	(       self     )					->				Dict:
													# The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
													# and special added tokens as tokens
													A__	     =							self.get_tokenizers(fast=__UpperCAmelCase		,do_lower_case=__UpperCAmelCase     )
													for tokenizer in tokenizers:
																				with self.subTest(f'''{tokenizer.__class__.__name__}'''     ):
																											A__	     =							['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
																											A__	     =							tokenizer.convert_tokens_to_string(__UpperCAmelCase     )

																											self.assertIsInstance(__UpperCAmelCase		,__UpperCAmelCase     )





						def 				snake_case__	(       self     )					->				Tuple:
													A__	     =							self.get_tokenizers()
													for tokenizer in tokenizers:
																				with self.subTest(f'''{tokenizer.__class__.__name__}'''     ):
																											A__	     =							[
																											    'bos_token',
																											    'eos_token',
																											    'unk_token',
																											    'sep_token',
																											    'pad_token',
																											    'cls_token',
																											    'mask_token',
																											]

																											A__	     =							0
																											A__	     =							tokenizer.convert_ids_to_tokens(
																											    __UpperCAmelCase		,skip_special_tokens=__UpperCAmelCase     )

																											for attr in attributes_list:
																																		setattr(__UpperCAmelCase		,attr + '_id'		,__UpperCAmelCase     )
																																		self.assertEqual(getattr(__UpperCAmelCase		,__UpperCAmelCase     )		,__UpperCAmelCase     )
																																		self.assertEqual(getattr(__UpperCAmelCase		,attr + '_id'     )		,__UpperCAmelCase     )

																																		setattr(__UpperCAmelCase		,attr + '_id'		,__UpperCAmelCase     )
																																		self.assertEqual(getattr(__UpperCAmelCase		,__UpperCAmelCase     )		,__UpperCAmelCase     )
																																		self.assertEqual(getattr(__UpperCAmelCase		,attr + '_id'     )		,__UpperCAmelCase     )

																											setattr(__UpperCAmelCase		,'additional_special_tokens_ids'		,[]     )
																											self.assertListEqual(getattr(__UpperCAmelCase		,'additional_special_tokens'     )		,[]     )
																											self.assertListEqual(getattr(__UpperCAmelCase		,'additional_special_tokens_ids'     )		,[]     )

																											setattr(__UpperCAmelCase		,'additional_special_tokens_ids'		,[token_id_to_test_setters]     )
																											self.assertListEqual(getattr(__UpperCAmelCase		,'additional_special_tokens'     )		,[token_to_test_setters]     )
																											self.assertListEqual(getattr(__UpperCAmelCase		,'additional_special_tokens_ids'     )		,[token_id_to_test_setters]     )





