






import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
    __a      :       int				  =   "pt"
elif is_tf_available():
    __a      :       int				  =   "tf"
else:
    __a      :       Optional[int]				  =   "jax"


class       __lowercase      (	UpperCAmelCase__			,      unittest.TestCase  ):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE   						=				ByTaTokenizer
     SCREAMING_SNAKE_CASE   						=				False




     def 		lowerCAmelCase_     (				self       :			Any				):


         """simple docstring"""



         super().setUp()
         __A         =	ByTaTokenizer()
         tokenizer.save_pretrained(self.tmpdirname				)




     @cached_property
     def 		lowerCAmelCase_     (				self       :			Optional[Any]				):


         """simple docstring"""



         return ByTaTokenizer.from_pretrained("""google/byt5-small"""				)




     def 		lowerCAmelCase_     (				self       :			List[Any] ,	**UpperCamelCase_       :			Any				):


         """simple docstring"""



         return self.tokenizer_class.from_pretrained(self.tmpdirname ,	**UpperCamelCase_				)




     def 		lowerCAmelCase_     (				self       :			str ,	UpperCamelCase_       :			List[Any] ,	UpperCamelCase_       :			Any=False ,	UpperCamelCase_       :			Optional[Any]=20 ,	UpperCamelCase_       :			Any=5				):


         """simple docstring"""



         __A         =	[]
         for i in range(len(UpperCamelCase_				)				):
             try:
                 __A         =	tokenizer.decode([i] ,	clean_up_tokenization_spaces=UpperCamelCase_				)
             except UnicodeDecodeError:
                 pass
             toks.append((i, tok)				)

         __A         =	list(filter(lambda UpperCamelCase_				: re.match(R"""^[ a-zA-Z]+$""" ,	t[1]				) ,	UpperCamelCase_				)				)
         __A         =	list(filter(lambda UpperCamelCase_				: [t[0]] == tokenizer.encode(t[1] ,	add_special_tokens=UpperCamelCase_				) ,	UpperCamelCase_				)				)
         if max_length is not None and len(UpperCamelCase_				) > max_length:
             __A         =	toks[:max_length]
         if min_length is not None and len(UpperCamelCase_				) < min_length and len(UpperCamelCase_				) > 0:
             while len(UpperCamelCase_				) < min_length:
                 __A         =	toks + toks
             # toks_str = [t[1] for t in toks]
         __A         =	[t[0] for t in toks]

         # Ensure consistency
         __A         =	tokenizer.decode(UpperCamelCase_ ,	clean_up_tokenization_spaces=UpperCamelCase_				)
         if " " not in output_txt and len(UpperCamelCase_				) > 1:
             __A         =	(
                 tokenizer.decode([toks_ids[0]] ,	clean_up_tokenization_spaces=UpperCamelCase_				)
                 + """ """
                 + tokenizer.decode(toks_ids[1:] ,	clean_up_tokenization_spaces=UpperCamelCase_				)
             )
         if with_prefix_space:
             __A         =	""" """ + output_txt
         __A         =	tokenizer.encode(UpperCamelCase_ ,	add_special_tokens=UpperCamelCase_				)
         return output_txt, output_ids




     def 		lowerCAmelCase_     (				self       :			Optional[int]				):


         """simple docstring"""



         __A         =	self.ta_base_tokenizer
         __A         =	tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]				)
         __A         =	tokenizer(["""hi""", """I went to the gym""", """"""]				)
         self.assertListEqual(batch_with_eos_added["""input_ids"""] ,	batch_without_eos_added["""input_ids"""]				)




     def 		lowerCAmelCase_     (				self       :			Any				):


         """simple docstring"""



         __A         =	self.ta_base_tokenizer
         __A         =	"""Unicode €."""
         __A         =	tokenizer(UpperCamelCase_				)
         __A         =	[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
         self.assertEqual(encoded["""input_ids"""] ,	UpperCamelCase_				)

         # decoding
         __A         =	tokenizer.decode(UpperCamelCase_				)
         self.assertEqual(UpperCamelCase_ ,	"""Unicode €.</s>"""				)

         __A         =	tokenizer("""e è é ê ë"""				)
         __A         =	[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
         self.assertEqual(encoded["""input_ids"""] ,	UpperCamelCase_				)
         # decoding
         __A         =	tokenizer.decode(UpperCamelCase_				)
         self.assertEqual(UpperCamelCase_ ,	"""e è é ê ë</s>"""				)

         # encode/decode, but with `encode` instead of `__call__`
         self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""				)				) ,	"""e è é ê ë</s>"""				)




     def 		lowerCAmelCase_     (				self       :			Tuple				):


         """simple docstring"""



         __A         =	self.ta_base_tokenizer
         __A         =	["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
         # fmt: off
         __A         =	[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
         # fmt: on
         __A         =	tokenizer(UpperCamelCase_ ,	padding=UpperCamelCase_ ,	return_tensors=UpperCamelCase_				)
         self.assertIsInstance(UpperCamelCase_ ,	UpperCamelCase_				)

         if FRAMEWORK != "jax":
             __A         =	list(batch.input_ids.numpy()[0]				)
         else:
             __A         =	list(batch.input_ids.tolist()[0]				)

         self.assertListEqual(UpperCamelCase_ ,	UpperCamelCase_				)

         self.assertEqual((2, 37) ,	batch.input_ids.shape				)
         self.assertEqual((2, 37) ,	batch.attention_mask.shape				)




     def 		lowerCAmelCase_     (				self       :			int				):


         """simple docstring"""



         __A         =	self.ta_base_tokenizer
         __A         =	["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
         __A         =	tokenizer(UpperCamelCase_ ,	padding=UpperCamelCase_ ,	return_tensors=UpperCamelCase_				)
         # check if input_ids are returned and no decoder_input_ids
         self.assertIn("""input_ids""" ,	UpperCamelCase_				)
         self.assertIn("""attention_mask""" ,	UpperCamelCase_				)
         self.assertNotIn("""decoder_input_ids""" ,	UpperCamelCase_				)
         self.assertNotIn("""decoder_attention_mask""" ,	UpperCamelCase_				)




     def 		lowerCAmelCase_     (				self       :			Optional[int]				):


         """simple docstring"""



         __A         =	self.ta_base_tokenizer
         __A         =	[
             """Summary of the text.""",
             """Another summary.""",
         ]
         __A         =	tokenizer(
             text_target=UpperCamelCase_ ,	max_length=32 ,	padding="""max_length""" ,	truncation=UpperCamelCase_ ,	return_tensors=UpperCamelCase_				)
         self.assertEqual(32 ,	targets["""input_ids"""].shape[1]				)




     def 		lowerCAmelCase_     (				self       :			Any				):


         """simple docstring"""



         __A         =	self.ta_base_tokenizer
         __A         =	["""A long paragraph for summarization. </s>"""]
         __A         =	["""Summary of the text. </s>"""]
         # fmt: off
         __A         =	[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
         __A         =	[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
         # fmt: on

         __A         =	tokenizer(UpperCamelCase_ ,	text_target=UpperCamelCase_				)

         self.assertEqual(UpperCamelCase_ ,	batch["""input_ids"""][0]				)
         self.assertEqual(UpperCamelCase_ ,	batch["""labels"""][0]				)




     def 		lowerCAmelCase_     (				self       :			Tuple				):


         """simple docstring"""



         __A         =	self.get_tokenizers()
         for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"				):
                 self.assertNotEqual(tokenizer.model_max_length ,	42				)

             # Now let's start the test
         __A         =	self.get_tokenizers()
         for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"				):
                 # Isolate this from the other tests because we save additional tokens/etc
                 __A         =	tempfile.mkdtemp()

                 __A         =	""" He is very happy, UNwant\u00E9d,running"""
                 __A         =	tokenizer.encode(UpperCamelCase_ ,	add_special_tokens=UpperCamelCase_				)
                 tokenizer.save_pretrained(UpperCamelCase_				)

                 __A         =	tokenizer.__class__.from_pretrained(UpperCamelCase_				)
                 __A         =	after_tokenizer.encode(UpperCamelCase_ ,	add_special_tokens=UpperCamelCase_				)
                 self.assertListEqual(UpperCamelCase_ ,	UpperCamelCase_				)

                 shutil.rmtree(UpperCamelCase_				)

         __A         =	self.get_tokenizers(model_max_length=42				)
         for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"				):
                 # Isolate this from the other tests because we save additional tokens/etc
                 __A         =	tempfile.mkdtemp()

                 __A         =	""" He is very happy, UNwant\u00E9d,running"""
                 tokenizer.add_tokens(["""bim""", """bambam"""]				)
                 __A         =	tokenizer.additional_special_tokens
                 additional_special_tokens.append("""new_additional_special_token"""				)
                 tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}				)
                 __A         =	tokenizer.encode(UpperCamelCase_ ,	add_special_tokens=UpperCamelCase_				)
                 tokenizer.save_pretrained(UpperCamelCase_				)

                 __A         =	tokenizer.__class__.from_pretrained(UpperCamelCase_				)
                 __A         =	after_tokenizer.encode(UpperCamelCase_ ,	add_special_tokens=UpperCamelCase_				)
                 self.assertListEqual(UpperCamelCase_ ,	UpperCamelCase_				)
                 self.assertIn("""new_additional_special_token""" ,	after_tokenizer.additional_special_tokens				)
                 self.assertEqual(after_tokenizer.model_max_length ,	42				)

                 __A         =	tokenizer.__class__.from_pretrained(UpperCamelCase_ ,	model_max_length=43				)
                 self.assertEqual(tokenizer.model_max_length ,	43				)

                 shutil.rmtree(UpperCamelCase_				)




     def 		lowerCAmelCase_     (				self       :			Dict				):


         """simple docstring"""



         __A         =	[]
         if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())				)

         if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())				)

         for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer_utils.save_pretrained(UpperCamelCase_				)

                 with open(os.path.join(UpperCamelCase_ ,	"""special_tokens_map.json"""				) ,	encoding="""utf-8"""				) as json_file:
                     __A         =	json.load(UpperCamelCase_				)

                 with open(os.path.join(UpperCamelCase_ ,	"""tokenizer_config.json"""				) ,	encoding="""utf-8"""				) as json_file:
                     __A         =	json.load(UpperCamelCase_				)

                 __A         =	[F"<extra_id_{i}>" for i in range(125				)]

                 __A         =	added_tokens_extra_ids + [
                     """an_additional_special_token"""
                 ]
                 __A         =	added_tokens_extra_ids + [
                     """an_additional_special_token"""
                 ]

                 with open(os.path.join(UpperCamelCase_ ,	"""special_tokens_map.json"""				) ,	"""w""" ,	encoding="""utf-8"""				) as outfile:
                     json.dump(UpperCamelCase_ ,	UpperCamelCase_				)
                 with open(os.path.join(UpperCamelCase_ ,	"""tokenizer_config.json"""				) ,	"""w""" ,	encoding="""utf-8"""				) as outfile:
                     json.dump(UpperCamelCase_ ,	UpperCamelCase_				)

                 # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                 # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                 # "special_tokens_map.json" files
                 __A         =	tokenizer_class.from_pretrained(
                     UpperCamelCase_ ,	)
                 self.assertIn(
                     """an_additional_special_token""" ,	tokenizer_without_change_in_init.additional_special_tokens				)
                 # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                 self.assertEqual(
                     ["""an_additional_special_token"""] ,	tokenizer_without_change_in_init.convert_ids_to_tokens(
                         tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]				)				) ,	)

                 # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                 __A         =	added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token""" ,	lstrip=UpperCamelCase_				)]
                 __A         =	tokenizer_class.from_pretrained(
                     UpperCamelCase_ ,	additional_special_tokens=UpperCamelCase_ ,	)

                 self.assertIn("""a_new_additional_special_token""" ,	tokenizer.additional_special_tokens				)
                 self.assertEqual(
                     ["""a_new_additional_special_token"""] ,	tokenizer.convert_ids_to_tokens(
                         tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]				)				) ,	)




     def 		lowerCAmelCase_     (				self       :			int				):


         """simple docstring"""



         __A         =	[]
         if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())				)

         if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())				)

         for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer_utils.save_pretrained(UpperCamelCase_				)

                 __A         =	tokenizer_class.from_pretrained(UpperCamelCase_				)

                 self.assertTrue(tokenizer.decode([255]				) == """"""				)




     def 		lowerCAmelCase_     (				self       :			Optional[Any]				):


         """simple docstring"""



         pass




     def 		lowerCAmelCase_     (				self       :			List[str]				):


         """simple docstring"""



         pass




     def 		lowerCAmelCase_     (				self       :			Union[str, Any]				):


         """simple docstring"""



         pass




     def 		lowerCAmelCase_     (				self       :			Any				):


         """simple docstring"""



         pass




     def 		lowerCAmelCase_     (				self       :			List[str]				):


         """simple docstring"""



         __A         =	self.get_tokenizers(fast=UpperCamelCase_ ,	do_lower_case=UpperCamelCase_				)
         for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"				):
                 __A         =	["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
                 __A         =	tokenizer.convert_tokens_to_string(UpperCamelCase_				)

                 self.assertIsInstance(UpperCamelCase_ ,	UpperCamelCase_				)





     def 		lowerCAmelCase_     (				self       :			Dict				):


         """simple docstring"""



         __A         =	self.get_tokenizers()
         for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"				):
                 __A         =	[
                     """bos_token""",
                     """eos_token""",
                     """unk_token""",
                     """sep_token""",
                     """pad_token""",
                     """cls_token""",
                     """mask_token""",
                 ]

                 __A         =	0
                 __A         =	tokenizer.convert_ids_to_tokens(
                     UpperCamelCase_ ,	skip_special_tokens=UpperCamelCase_				)

                 for attr in attributes_list:
                     setattr(UpperCamelCase_ ,	attr + """_id""" ,	UpperCamelCase_				)
                     self.assertEqual(getattr(UpperCamelCase_ ,	UpperCamelCase_				) ,	UpperCamelCase_				)
                     self.assertEqual(getattr(UpperCamelCase_ ,	attr + """_id"""				) ,	UpperCamelCase_				)

                     setattr(UpperCamelCase_ ,	attr + """_id""" ,	UpperCamelCase_				)
                     self.assertEqual(getattr(UpperCamelCase_ ,	UpperCamelCase_				) ,	UpperCamelCase_				)
                     self.assertEqual(getattr(UpperCamelCase_ ,	attr + """_id"""				) ,	UpperCamelCase_				)

                 setattr(UpperCamelCase_ ,	"""additional_special_tokens_ids""" ,	[]				)
                 self.assertListEqual(getattr(UpperCamelCase_ ,	"""additional_special_tokens"""				) ,	[]				)
                 self.assertListEqual(getattr(UpperCamelCase_ ,	"""additional_special_tokens_ids"""				) ,	[]				)

                 setattr(UpperCamelCase_ ,	"""additional_special_tokens_ids""" ,	[token_id_to_test_setters]				)
                 self.assertListEqual(getattr(UpperCamelCase_ ,	"""additional_special_tokens"""				) ,	[token_to_test_setters]				)
                 self.assertListEqual(getattr(UpperCamelCase_ ,	"""additional_special_tokens_ids"""				) ,	[token_id_to_test_setters]				)












from dataclasses import dataclass
from typing import Optional, Tuple, Union

import torch
import torch.nn as nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput, apply_forward_hook
from .modeling_utils import ModelMixin
from .vae import Decoder, DecoderOutput, Encoder, VectorQuantizer


@dataclass
class       __lowercase      (	lowercase_  ):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE   						=				42


class       __lowercase      (	lowercase_			,      lowercase_  ):







     '''simple docstring'''




     @register_to_config
     def __init__(				self       :			List[Any] ,	UpperCamelCase_       :			int = 3 ,	UpperCamelCase_       :			int = 3 ,	UpperCamelCase_       :			Tuple[str] = ("DownEncoderBlock2D",) ,	UpperCamelCase_       :			Tuple[str] = ("UpDecoderBlock2D",) ,	UpperCamelCase_       :			Tuple[int] = (64,) ,	UpperCamelCase_       :			int = 1 ,	UpperCamelCase_       :			str = "silu" ,	UpperCamelCase_       :			int = 3 ,	UpperCamelCase_       :			int = 32 ,	UpperCamelCase_       :			int = 256 ,	UpperCamelCase_       :			int = 32 ,	UpperCamelCase_       :			Optional[int] = None ,	UpperCamelCase_       :			float = 0.18215 ,	UpperCamelCase_       :			str = "group" ,	):


         """simple docstring"""



         super().__init__()

         # pass init params to Encoder
         __A         =	Encoder(
             in_channels=UpperCamelCase_ ,	out_channels=UpperCamelCase_ ,	down_block_types=UpperCamelCase_ ,	block_out_channels=UpperCamelCase_ ,	layers_per_block=UpperCamelCase_ ,	act_fn=UpperCamelCase_ ,	norm_num_groups=UpperCamelCase_ ,	double_z=UpperCamelCase_ ,	)

         __A         =	vq_embed_dim if vq_embed_dim is not None else latent_channels

         __A         =	nn.Convad(UpperCamelCase_ ,	UpperCamelCase_ ,	1				)
         __A         =	VectorQuantizer(UpperCamelCase_ ,	UpperCamelCase_ ,	beta=0.25 ,	remap=UpperCamelCase_ ,	sane_index_shape=UpperCamelCase_				)
         __A         =	nn.Convad(UpperCamelCase_ ,	UpperCamelCase_ ,	1				)

         # pass init params to Decoder
         __A         =	Decoder(
             in_channels=UpperCamelCase_ ,	out_channels=UpperCamelCase_ ,	up_block_types=UpperCamelCase_ ,	block_out_channels=UpperCamelCase_ ,	layers_per_block=UpperCamelCase_ ,	act_fn=UpperCamelCase_ ,	norm_num_groups=UpperCamelCase_ ,	norm_type=UpperCamelCase_ ,	)




     @apply_forward_hook
     def 		lowerCAmelCase_     (				self       :			List[str] ,	UpperCamelCase_       :			torch.FloatTensor ,	UpperCamelCase_       :			bool = True				):


         """simple docstring"""



         __A         =	self.encoder(UpperCamelCase_				)
         __A         =	self.quant_conv(UpperCamelCase_				)

         if not return_dict:
             return (h,)

         return VQEncoderOutput(latents=UpperCamelCase_				)




     @apply_forward_hook
     def 		lowerCAmelCase_     (				self       :			List[str] ,	UpperCamelCase_       :			torch.FloatTensor ,	UpperCamelCase_       :			bool = False ,	UpperCamelCase_       :			bool = True				):


         """simple docstring"""



         if not force_not_quantize:
             __A    ,	__A    ,	__A         =	self.quantize(UpperCamelCase_				)
         else:
             __A         =	h
         __A         =	self.post_quant_conv(UpperCamelCase_				)
         __A         =	self.decoder(UpperCamelCase_ ,	quant if self.config.norm_type == """spatial""" else None				)

         if not return_dict:
             return (dec,)

         return DecoderOutput(sample=UpperCamelCase_				)





     def 		lowerCAmelCase_     (				self       :			Any ,	UpperCamelCase_       :			torch.FloatTensor ,	UpperCamelCase_       :			bool = True				):


         """simple docstring"""



         __A         =	sample
         __A         =	self.encode(UpperCamelCase_				).latents
         __A         =	self.decode(UpperCamelCase_				).sample

         if not return_dict:
             return (dec,)

         return DecoderOutput(sample=UpperCamelCase_				)




