





from typing import List, Optional, Union

import torch

from ...models import UNetaDConditionModel, VQModel
from ...pipelines import DiffusionPipeline
from ...pipelines.pipeline_utils import ImagePipelineOutput
from ...schedulers import DDPMScheduler
from ...utils import (
    is_accelerate_available,
    is_accelerate_version,
    logging,
    randn_tensor,
    replace_example_docstring,
)


lowerCAmelCase_        =							logging.get_logger(__name__)  # pylint: disable=invalid-name

lowerCAmelCase_        =							'''
    Examples:
        ```py
        >>> from diffusers import KandinskyV22Pipeline, KandinskyV22PriorPipeline
        >>> import torch

        >>> pipe_prior = KandinskyV22PriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-prior")
        >>> pipe_prior.to("cuda")
        >>> prompt = "red cat, 4k photo"
        >>> out = pipe_prior(prompt)
        >>> image_emb = out.image_embeds
        >>> zero_image_emb = out.negative_image_embeds
        >>> pipe = KandinskyV22Pipeline.from_pretrained("kandinsky-community/kandinsky-2-2-decoder")
        >>> pipe.to("cuda")
        >>> image = pipe(
        ...     image_embeds=image_emb,
        ...     negative_image_embeds=zero_image_emb,
        ...     height=768,
        ...     width=768,
        ...     num_inference_steps=50,
        ... ).images
        >>> image[0].save("cat.png")
        ```
'''






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase=8					)						->		Union[str, Any]:






				"""simple docstring"""


				snake_case_		:       int           =    height // scale_factor**2
				if height % scale_factor**2 != 0:
								new_height += 1
				snake_case_		:       int           =    width // scale_factor**2
				if width % scale_factor**2 != 0:
								new_width += 1
				return new_height * scale_factor, new_width * scale_factor





class    __lowerCAmelCase			(     _a	):






			def __init__(self   ,				__magic_name__   ,				__magic_name__   ,				__magic_name__   ,				)	->  str:


							'''simple docstring'''



							super().__init__()

							self.register_modules(
							    unet=__magic_name__   ,				scheduler=__magic_name__   ,				movq=__magic_name__   ,				)
							snake_case_		:       List[str]           =    2 ** (len(self.movq.config.block_out_channels       ) - 1)






			def      lowerCamelCase    (self   ,				__magic_name__   ,				__magic_name__   ,				__magic_name__   ,				__magic_name__   ,				__magic_name__   ,				__magic_name__       )	->  str:


							'''simple docstring'''



							if latents is None:
											snake_case_		:       int           =    randn_tensor(__magic_name__   ,				generator=__magic_name__   ,				device=__magic_name__   ,				dtype=__magic_name__       )
							else:
											if latents.shape != shape:
															raise ValueError(F'''Unexpected latents shape, got {latents.shape}, expected {shape}'''       )
											snake_case_		:       int           =    latents.to(__magic_name__       )

							snake_case_		:       Any           =    latents * scheduler.init_noise_sigma
							return latents






			def      lowerCamelCase    (self   ,				__magic_name__=0       )	->  Any:


							'''simple docstring'''



							if is_accelerate_available():
											from accelerate import cpu_offload
							else:
											raise ImportError('''Please install accelerate via `pip install accelerate`'''       )

							snake_case_		:       Optional[int]           =    torch.device(F'''cuda:{gpu_id}'''       )

							snake_case_		:       Optional[int]           =    [
							    self.unet,
							    self.movq,
							]
							for cpu_offloaded_model in models:
											if cpu_offloaded_model is not None:
															cpu_offload(__magic_name__   ,				__magic_name__       )






			def      lowerCamelCase    (self   ,				__magic_name__=0       )	->  int:


							'''simple docstring'''



							if is_accelerate_available() and is_accelerate_version('''>='''   ,				'''0.17.0.dev0'''       ):
											from accelerate import cpu_offload_with_hook
							else:
											raise ImportError('''`enable_model_cpu_offload` requires `accelerate v0.17.0` or higher.'''       )

							snake_case_		:       Tuple           =    torch.device(F'''cuda:{gpu_id}'''       )

							if self.device.type != "cpu":
											self.to('''cpu'''   ,				silence_dtype_warnings=__magic_name__       )
											torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)

							snake_case_		:       Tuple           =    None
							for cpu_offloaded_model in [self.unet, self.movq]:
											snake_case_     ,				snake_case_		:       str           =    cpu_offload_with_hook(__magic_name__   ,				__magic_name__   ,				prev_module_hook=__magic_name__       )

							# We'll offload the last model manually.
							snake_case_		:       str           =    hook






			@property
			# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._execution_device
			def      lowerCamelCase    (self       )	->  Optional[int]:


							'''simple docstring'''



							if not hasattr(self.unet   ,				'''_hf_hook'''       ):
											return self.device
							for module in self.unet.modules():
											if (
											    hasattr(__magic_name__   ,				'''_hf_hook'''       )
											    and hasattr(module._hf_hook   ,				'''execution_device'''       )
											    and module._hf_hook.execution_device is not None
											):
															return torch.device(module._hf_hook.execution_device       )
							return self.device






			@torch.no_grad()
			@replace_example_docstring(__magic_name__       )
			def __call__(self   ,				__magic_name__   ,				__magic_name__   ,				__magic_name__ = 512   ,				__magic_name__ = 512   ,				__magic_name__ = 100   ,				__magic_name__ = 4.0   ,				__magic_name__ = 1   ,				__magic_name__ = None   ,				__magic_name__ = None   ,				__magic_name__ = "pil"   ,				__magic_name__ = True   ,				)	->  List[Any]:


							'''simple docstring'''



							snake_case_		:       List[str]           =    self._execution_device

							snake_case_		:       Optional[Any]           =    guidance_scale > 1.0

							if isinstance(__magic_name__   ,				__magic_name__       ):
											snake_case_		:       Tuple           =    torch.cat(__magic_name__   ,				dim=0       )
							snake_case_		:       Union[str, Any]           =    image_embeds.shape[0] * num_images_per_prompt
							if isinstance(__magic_name__   ,				__magic_name__       ):
											snake_case_		:       Dict           =    torch.cat(__magic_name__   ,				dim=0       )

							if do_classifier_free_guidance:
											snake_case_		:       List[str]           =    image_embeds.repeat_interleave(__magic_name__   ,				dim=0       )
											snake_case_		:       List[Any]           =    negative_image_embeds.repeat_interleave(__magic_name__   ,				dim=0       )

							snake_case_		:       str           =    torch.cat([negative_image_embeds, image_embeds]   ,				dim=0       ).to(dtype=self.unet.dtype   ,				device=__magic_name__       )

							self.scheduler.set_timesteps(__magic_name__   ,				device=__magic_name__       )
							snake_case_		:       Tuple           =    self.scheduler.timesteps

							snake_case_		:       Any           =    self.unet.config.in_channels

							snake_case_     ,				snake_case_		:       Union[str, Any]           =    downscale_height_and_width(__magic_name__   ,				__magic_name__   ,				self.movq_scale_factor       )

							# create initial latent
							snake_case_		:       List[Any]           =    self.prepare_latents(
							    (batch_size, num_channels_latents, height, width)   ,				image_embeds.dtype   ,				__magic_name__   ,				__magic_name__   ,				__magic_name__   ,				self.scheduler   ,				)

							for i, t in enumerate(self.progress_bar(__magic_name__       )       ):
											# expand the latents if we are doing classifier free guidance
											snake_case_		:       Any           =    torch.cat([latents] * 2       ) if do_classifier_free_guidance else latents

											snake_case_		:       List[Any]           =    {'''image_embeds''': image_embeds}
											snake_case_		:       Union[str, Any]           =    self.unet(
											    sample=__magic_name__   ,				timestep=__magic_name__   ,				encoder_hidden_states=__magic_name__   ,				added_cond_kwargs=__magic_name__   ,				return_dict=__magic_name__   ,				)[0]

											if do_classifier_free_guidance:
															snake_case_     ,				snake_case_		:       List[str]           =    noise_pred.split(latents.shape[1]   ,				dim=1       )
															snake_case_     ,				snake_case_		:       Union[str, Any]           =    noise_pred.chunk(2       )
															snake_case_     ,				snake_case_		:       Dict           =    variance_pred.chunk(2       )
															snake_case_		:       Dict           =    noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
															snake_case_		:       Optional[int]           =    torch.cat([noise_pred, variance_pred_text]   ,				dim=1       )

											if not (
											    hasattr(self.scheduler.config   ,				'''variance_type'''       )
											    and self.scheduler.config.variance_type in ["learned", "learned_range"]
											):
															snake_case_     ,				snake_case_		:       int           =    noise_pred.split(latents.shape[1]   ,				dim=1       )

											# compute the previous noisy sample x_t -> x_t-1
											snake_case_		:       List[Any]           =    self.scheduler.step(
											    __magic_name__   ,				__magic_name__   ,				__magic_name__   ,				generator=__magic_name__   ,				)[0]
							# post-processing
							snake_case_		:       int           =    self.movq.decode(__magic_name__   ,				force_not_quantize=__magic_name__       )['''sample''']

							if output_type not in ["pt", "np", "pil"]:
											raise ValueError(F'''Only the output types `pt`, `pil` and `np` are supported not output_type={output_type}'''       )

							if output_type in ["np", "pil"]:
											snake_case_		:       Tuple           =    image * 0.5 + 0.5
											snake_case_		:       List[str]           =    image.clamp(0   ,				1       )
											snake_case_		:       int           =    image.cpu().permute(0   ,				2   ,				3   ,				1       ).float().numpy()

							if output_type == "pil":
											snake_case_		:       str           =    self.numpy_to_pil(__magic_name__       )

							if not return_dict:
											return (image,)

							return ImagePipelineOutput(images=__magic_name__       )








import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
			UpperCAmelCase__      			=						'''pt'''
elif is_tf_available():
			UpperCAmelCase__      			=						'''tf'''
else:
			UpperCAmelCase__      			=						'''jax'''




class 							snake_case_       (    __UpperCamelCase		,							unittest.TestCase      ):

						"""simple docstring"""



						snake_case__        =      ByTaTokenizer
						snake_case__        =      False



						def        UpperCAmelCase__					(self:					int   )    ->     str:




											'''simple docstring'''



											super().setUp()
											__a					:						int      =					ByTaTokenizer()
											tokenizer.save_pretrained(self.tmpdirname   )



						@cached_property
						def        UpperCAmelCase__					(self:					Any   )    ->     Optional[int]:




											'''simple docstring'''



											return ByTaTokenizer.from_pretrained("google/byt5-small"   )



						def        UpperCAmelCase__					(self:					List[str] ,					**__UpperCAmelCase:					Dict   )    ->     ByTaTokenizer:




											'''simple docstring'''



											return self.tokenizer_class.from_pretrained(self.tmpdirname ,					**__UpperCAmelCase   )



						def        UpperCAmelCase__					(self:					int ,					__UpperCAmelCase:					Tuple ,					__UpperCAmelCase:					Optional[int]=False ,					__UpperCAmelCase:					Optional[Any]=20 ,					__UpperCAmelCase:					Tuple=5   )    ->     Tuple[str, list]:




											'''simple docstring'''



											__a					:						Tuple      =					[]
											for i in range(len(__UpperCAmelCase   )   ):
																try:
																					__a					:						Union[str, Any]      =					tokenizer.decode([i] ,					clean_up_tokenization_spaces=__UpperCAmelCase   )
																except UnicodeDecodeError:
																					pass
																toks.append((i, tok)   )

											__a					:						Any      =					list(filter(lambda __UpperCAmelCase   : re.match(R"^[ a-zA-Z]+$" ,					t[1]   ) ,					__UpperCAmelCase   )   )
											__a					:						int      =					list(filter(lambda __UpperCAmelCase   : [t[0]] == tokenizer.encode(t[1] ,					add_special_tokens=__UpperCAmelCase   ) ,					__UpperCAmelCase   )   )
											if max_length is not None and len(__UpperCAmelCase   ) > max_length:
																__a					:						str      =					toks[:max_length]
											if min_length is not None and len(__UpperCAmelCase   ) < min_length and len(__UpperCAmelCase   ) > 0:
																while len(__UpperCAmelCase   ) < min_length:
																					__a					:						List[str]      =					toks + toks
        # toks_str = [t[1] for t in toks]
											__a					:						Optional[int]      =					[t[0] for t in toks]

											# Ensure consistency
											__a					:						Any      =					tokenizer.decode(__UpperCAmelCase ,					clean_up_tokenization_spaces=__UpperCAmelCase   )
											if " " not in output_txt and len(__UpperCAmelCase   ) > 1:
																__a					:						Union[str, Any]      =					(
																    tokenizer.decode([toks_ids[0]] ,					clean_up_tokenization_spaces=__UpperCAmelCase   )
																    + " "
																    + tokenizer.decode(toks_ids[1:] ,					clean_up_tokenization_spaces=__UpperCAmelCase   )
																)
											if with_prefix_space:
																__a					:						Tuple      =					" " + output_txt
											__a					:						Tuple      =					tokenizer.encode(__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase   )
											return output_txt, output_ids



						def        UpperCAmelCase__					(self:					Tuple   )    ->     List[Any]:




											'''simple docstring'''



											__a					:						Any      =					self.ta_base_tokenizer
											__a					:						str      =					tokenizer(["hi</s>", "I went to the gym</s>", "</s>"]   )
											__a					:						int      =					tokenizer(["hi", "I went to the gym", ""]   )
											self.assertListEqual(batch_with_eos_added["input_ids"] ,					batch_without_eos_added["input_ids"]   )



						def        UpperCAmelCase__					(self:					List[str]   )    ->     Union[str, Any]:




											'''simple docstring'''



											__a					:						Any      =					self.ta_base_tokenizer
											__a					:						int      =					"Unicode €."
											__a					:						int      =					tokenizer(__UpperCAmelCase   )
											__a					:						List[Any]      =					[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
											self.assertEqual(encoded["input_ids"] ,					__UpperCAmelCase   )

											# decoding
											__a					:						List[str]      =					tokenizer.decode(__UpperCAmelCase   )
											self.assertEqual(__UpperCAmelCase ,					"Unicode €.</s>"   )

											__a					:						Optional[int]      =					tokenizer("e è é ê ë"   )
											__a					:						List[str]      =					[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
											self.assertEqual(encoded["input_ids"] ,					__UpperCAmelCase   )
											# decoding
											__a					:						Optional[Any]      =					tokenizer.decode(__UpperCAmelCase   )
											self.assertEqual(__UpperCAmelCase ,					"e è é ê ë</s>"   )

											# encode/decode, but with `encode` instead of `__call__`
											self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"   )   ) ,					"e è é ê ë</s>"   )



						def        UpperCAmelCase__					(self:					List[str]   )    ->     Optional[int]:




											'''simple docstring'''



											__a					:						Optional[Any]      =					self.ta_base_tokenizer
											__a					:						Any      =					["A long paragraph for summarization.", "Another paragraph for summarization."]
											# fmt: off
											__a					:						Optional[int]      =					[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
											# fmt: on
											__a					:						Optional[Any]      =					tokenizer(__UpperCAmelCase ,					padding=__UpperCAmelCase ,					return_tensors=__UpperCAmelCase   )
											self.assertIsInstance(__UpperCAmelCase ,					__UpperCAmelCase   )

											if FRAMEWORK != "jax":
																__a					:						Optional[int]      =					list(batch.input_ids.numpy()[0]   )
											else:
																__a					:						List[Any]      =					list(batch.input_ids.tolist()[0]   )

											self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase   )

											self.assertEqual((2, 37) ,					batch.input_ids.shape   )
											self.assertEqual((2, 37) ,					batch.attention_mask.shape   )



						def        UpperCAmelCase__					(self:					Union[str, Any]   )    ->     Union[str, Any]:




											'''simple docstring'''



											__a					:						int      =					self.ta_base_tokenizer
											__a					:						str      =					["A long paragraph for summarization.", "Another paragraph for summarization."]
											__a					:						Optional[Any]      =					tokenizer(__UpperCAmelCase ,					padding=__UpperCAmelCase ,					return_tensors=__UpperCAmelCase   )
											# check if input_ids are returned and no decoder_input_ids
											self.assertIn("input_ids" ,					__UpperCAmelCase   )
											self.assertIn("attention_mask" ,					__UpperCAmelCase   )
											self.assertNotIn("decoder_input_ids" ,					__UpperCAmelCase   )
											self.assertNotIn("decoder_attention_mask" ,					__UpperCAmelCase   )



						def        UpperCAmelCase__					(self:					Union[str, Any]   )    ->     str:




											'''simple docstring'''



											__a					:						List[str]      =					self.ta_base_tokenizer
											__a					:						Optional[int]      =					[
											    "Summary of the text.",
											    "Another summary.",
											]
											__a					:						Union[str, Any]      =					tokenizer(
											    text_target=__UpperCAmelCase ,					max_length=32 ,					padding="max_length" ,					truncation=__UpperCAmelCase ,					return_tensors=__UpperCAmelCase   )
											self.assertEqual(32 ,					targets["input_ids"].shape[1]   )



						def        UpperCAmelCase__					(self:					str   )    ->     str:




											'''simple docstring'''



											__a					:						Union[str, Any]      =					self.ta_base_tokenizer
											__a					:						Union[str, Any]      =					["A long paragraph for summarization. </s>"]
											__a					:						Dict      =					["Summary of the text. </s>"]
											# fmt: off
											__a					:						str      =					[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
											__a					:						Union[str, Any]      =					[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
											# fmt: on

											__a					:						int      =					tokenizer(__UpperCAmelCase ,					text_target=__UpperCAmelCase   )

											self.assertEqual(__UpperCAmelCase ,					batch["input_ids"][0]   )
											self.assertEqual(__UpperCAmelCase ,					batch["labels"][0]   )



						def        UpperCAmelCase__					(self:					Union[str, Any]   )    ->     Tuple:




											'''simple docstring'''



											__a					:						Optional[Any]      =					self.get_tokenizers()
											for tokenizer in tokenizers:
																with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																					self.assertNotEqual(tokenizer.model_max_length ,					42   )

        # Now let's start the test
											__a					:						str      =					self.get_tokenizers()
											for tokenizer in tokenizers:
																with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																					# Isolate this from the other tests because we save additional tokens/etc
																					__a					:						int      =					tempfile.mkdtemp()

																					__a					:						Optional[int]      =					" He is very happy, UNwant\u00E9d,running"
																					__a					:						str      =					tokenizer.encode(__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase   )
																					tokenizer.save_pretrained(__UpperCAmelCase   )

																					__a					:						int      =					tokenizer.__class__.from_pretrained(__UpperCAmelCase   )
																					__a					:						Tuple      =					after_tokenizer.encode(__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase   )
																					self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase   )

																					shutil.rmtree(__UpperCAmelCase   )

											__a					:						List[str]      =					self.get_tokenizers(model_max_length=42   )
											for tokenizer in tokenizers:
																with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																					# Isolate this from the other tests because we save additional tokens/etc
																					__a					:						Optional[int]      =					tempfile.mkdtemp()

																					__a					:						Any      =					" He is very happy, UNwant\u00E9d,running"
																					tokenizer.add_tokens(["bim", "bambam"]   )
																					__a					:						List[str]      =					tokenizer.additional_special_tokens
																					additional_special_tokens.append("new_additional_special_token"   )
																					tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}   )
																					__a					:						Tuple      =					tokenizer.encode(__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase   )
																					tokenizer.save_pretrained(__UpperCAmelCase   )

																					__a					:						Optional[Any]      =					tokenizer.__class__.from_pretrained(__UpperCAmelCase   )
																					__a					:						List[str]      =					after_tokenizer.encode(__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase   )
																					self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase   )
																					self.assertIn("new_additional_special_token" ,					after_tokenizer.additional_special_tokens   )
																					self.assertEqual(after_tokenizer.model_max_length ,					42   )

																					__a					:						List[Any]      =					tokenizer.__class__.from_pretrained(__UpperCAmelCase ,					model_max_length=43   )
																					self.assertEqual(tokenizer.model_max_length ,					43   )

																					shutil.rmtree(__UpperCAmelCase   )



						def        UpperCAmelCase__					(self:					Optional[int]   )    ->     Tuple:




											'''simple docstring'''



											__a					:						Tuple      =					[]
											if self.test_slow_tokenizer:
																tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())   )

											if self.test_rust_tokenizer:
																tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())   )

											for tokenizer_class, tokenizer_utils in tokenizer_list:
																with tempfile.TemporaryDirectory() as tmp_dir:
																					tokenizer_utils.save_pretrained(__UpperCAmelCase   )

																					with open(os.path.join(__UpperCAmelCase ,					"special_tokens_map.json"   ) ,					encoding="utf-8"   ) as json_file:
																										__a					:						Optional[int]      =					json.load(__UpperCAmelCase   )

																					with open(os.path.join(__UpperCAmelCase ,					"tokenizer_config.json"   ) ,					encoding="utf-8"   ) as json_file:
																										__a					:						Any      =					json.load(__UpperCAmelCase   )

																					__a					:						Optional[Any]      =					[f'<extra_id_{i}>' for i in range(125   )]

																					__a					:						Any      =					added_tokens_extra_ids + [
																					    "an_additional_special_token"
																					]
																					__a					:						Union[str, Any]      =					added_tokens_extra_ids + [
																					    "an_additional_special_token"
																					]

																					with open(os.path.join(__UpperCAmelCase ,					"special_tokens_map.json"   ) ,					"w" ,					encoding="utf-8"   ) as outfile:
																										json.dump(__UpperCAmelCase ,					__UpperCAmelCase   )
																					with open(os.path.join(__UpperCAmelCase ,					"tokenizer_config.json"   ) ,					"w" ,					encoding="utf-8"   ) as outfile:
																										json.dump(__UpperCAmelCase ,					__UpperCAmelCase   )

																					# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																					# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																					# "special_tokens_map.json" files
																					__a					:						Union[str, Any]      =					tokenizer_class.from_pretrained(
																					    __UpperCAmelCase ,					)
																					self.assertIn(
																					    "an_additional_special_token" ,					tokenizer_without_change_in_init.additional_special_tokens   )
																					# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																					self.assertEqual(
																					    ["an_additional_special_token"] ,					tokenizer_without_change_in_init.convert_ids_to_tokens(
																					        tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]   )   ) ,					)

																					# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																					__a					:						Union[str, Any]      =					added_tokens_extra_ids + [AddedToken("a_new_additional_special_token" ,					lstrip=__UpperCAmelCase   )]
																					__a					:						int      =					tokenizer_class.from_pretrained(
																					    __UpperCAmelCase ,					additional_special_tokens=__UpperCAmelCase ,					)

																					self.assertIn("a_new_additional_special_token" ,					tokenizer.additional_special_tokens   )
																					self.assertEqual(
																					    ["a_new_additional_special_token"] ,					tokenizer.convert_ids_to_tokens(
																					        tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]   )   ) ,					)



						def        UpperCAmelCase__					(self:					List[str]   )    ->     List[str]:




											'''simple docstring'''



											__a					:						Optional[int]      =					[]
											if self.test_slow_tokenizer:
																tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())   )

											if self.test_rust_tokenizer:
																tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())   )

											for tokenizer_class, tokenizer_utils in tokenizer_list:
																with tempfile.TemporaryDirectory() as tmp_dir:
																					tokenizer_utils.save_pretrained(__UpperCAmelCase   )

																					__a					:						Tuple      =					tokenizer_class.from_pretrained(__UpperCAmelCase   )

																					self.assertTrue(tokenizer.decode([255]   ) == ""   )



						def        UpperCAmelCase__					(self:					Optional[int]   )    ->     str:




											'''simple docstring'''



											pass



						def        UpperCAmelCase__					(self:					str   )    ->     str:




											'''simple docstring'''



											pass



						def        UpperCAmelCase__					(self:					List[Any]   )    ->     Any:




											'''simple docstring'''



											pass



						def        UpperCAmelCase__					(self:					Dict   )    ->     Union[str, Any]:




											'''simple docstring'''



											pass



						def        UpperCAmelCase__					(self:					Dict   )    ->     int:




											'''simple docstring'''



											__a					:						Optional[int]      =					self.get_tokenizers(fast=__UpperCAmelCase ,					do_lower_case=__UpperCAmelCase   )
											for tokenizer in tokenizers:
																with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																					__a					:						List[str]      =					["t", "h", "i", "s", " ", "i", "s", " ", "a", " ", "t", "e", "x", "t", "</s>"]
																					__a					:						Any      =					tokenizer.convert_tokens_to_string(__UpperCAmelCase   )

																					self.assertIsInstance(__UpperCAmelCase ,					__UpperCAmelCase   )



						def        UpperCAmelCase__					(self:					List[Any]   )    ->     Any:




											'''simple docstring'''



											__a					:						Union[str, Any]      =					self.get_tokenizers()
											for tokenizer in tokenizers:
																with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																					__a					:						Tuple      =					[
																					    "bos_token",
																					    "eos_token",
																					    "unk_token",
																					    "sep_token",
																					    "pad_token",
																					    "cls_token",
																					    "mask_token",
																					]

																					__a					:						int      =					0
																					__a					:						Optional[Any]      =					tokenizer.convert_ids_to_tokens(
																					    __UpperCAmelCase ,					skip_special_tokens=__UpperCAmelCase   )

																					for attr in attributes_list:
																										setattr(__UpperCAmelCase ,					attr + "_id" ,					__UpperCAmelCase   )
																										self.assertEqual(getattr(__UpperCAmelCase ,					__UpperCAmelCase   ) ,					__UpperCAmelCase   )
																										self.assertEqual(getattr(__UpperCAmelCase ,					attr + "_id"   ) ,					__UpperCAmelCase   )

																										setattr(__UpperCAmelCase ,					attr + "_id" ,					__UpperCAmelCase   )
																										self.assertEqual(getattr(__UpperCAmelCase ,					__UpperCAmelCase   ) ,					__UpperCAmelCase   )
																										self.assertEqual(getattr(__UpperCAmelCase ,					attr + "_id"   ) ,					__UpperCAmelCase   )

																					setattr(__UpperCAmelCase ,					"additional_special_tokens_ids" ,					[]   )
																					self.assertListEqual(getattr(__UpperCAmelCase ,					"additional_special_tokens"   ) ,					[]   )
																					self.assertListEqual(getattr(__UpperCAmelCase ,					"additional_special_tokens_ids"   ) ,					[]   )

																					setattr(__UpperCAmelCase ,					"additional_special_tokens_ids" ,					[token_id_to_test_setters]   )
																					self.assertListEqual(getattr(__UpperCAmelCase ,					"additional_special_tokens"   ) ,					[token_to_test_setters]   )
																					self.assertListEqual(getattr(__UpperCAmelCase ,					"additional_special_tokens_ids"   ) ,					[token_id_to_test_setters]   )





