


import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
 lowerCamelCase			:					Optional[int]         =	'''pt'''
elif is_tf_available():
 lowerCamelCase			:					Union[str, Any]         =	'''tf'''
else:
 lowerCamelCase			:					Union[str, Any]         =	'''jax'''



class 							_UpperCamelCase			(_snake_case							,		unittest.TestCase			):
      snake_case_  =							ByTaTokenizer
      snake_case_  =							False


      def  __UpperCAmelCase		(	self	)->	Any:
           super().setUp()
           __lowerCAmelCase								=					ByTaTokenizer()
           tokenizer.save_pretrained(self.tmpdirname	)


      @cached_property
      def  __UpperCAmelCase		(	self	)->	Optional[int]:
           return ByTaTokenizer.from_pretrained("google/byt5-small"	)


      def  __UpperCAmelCase		(	self					,						**__UpperCamelCase	)->	ByTaTokenizer:
           return self.tokenizer_class.from_pretrained(self.tmpdirname					,						**lowerCAmelCase__	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase=False					,						__UpperCamelCase=2_0					,						__UpperCamelCase=5	)->	Tuple[str, list]:
           # XXX The default common tokenizer tests assume that every ID is decodable on its own.
           # This assumption is invalid for ByT5 because single bytes might not be
           # valid utf-8 (byte 128 for instance).
           # Here we're overriding the smallest possible method to provide
           # a clean sequence without making the same assumption.

           __lowerCAmelCase								=					[]
           for i in range(len(lowerCAmelCase__	)	):
                try:
                     __lowerCAmelCase								=					tokenizer.decode([i]					,						clean_up_tokenization_spaces=lowerCAmelCase__	)
                except UnicodeDecodeError:
                     pass
                toks.append((i, tok)	)

           __lowerCAmelCase								=					list(filter(lambda __UpperCamelCase	: re.match(R"^[ a-zA-Z]+$"					,						t[1]	)					,						lowerCAmelCase__	)	)
           __lowerCAmelCase								=					list(filter(lambda __UpperCamelCase	: [t[0]] == tokenizer.encode(t[1]					,						add_special_tokens=lowerCAmelCase__	)					,						lowerCAmelCase__	)	)
           if max_length is not None and len(lowerCAmelCase__	) > max_length:
                __lowerCAmelCase								=					toks[:max_length]
           if min_length is not None and len(lowerCAmelCase__	) < min_length and len(lowerCAmelCase__	) > 0:
                while len(lowerCAmelCase__	) < min_length:
                     __lowerCAmelCase								=					toks + toks
                      # toks_str = [t[1] for t in toks]
           __lowerCAmelCase								=					[t[0] for t in toks]

           # Ensure consistency
           __lowerCAmelCase								=					tokenizer.decode(lowerCAmelCase__					,						clean_up_tokenization_spaces=lowerCAmelCase__	)
           if " " not in output_txt and len(lowerCAmelCase__	) > 1:
                __lowerCAmelCase								=					(
                    tokenizer.decode([toks_ids[0]]					,						clean_up_tokenization_spaces=lowerCAmelCase__	)
                    + " "
                    + tokenizer.decode(toks_ids[1:]					,						clean_up_tokenization_spaces=lowerCAmelCase__	)
                )
           if with_prefix_space:
                __lowerCAmelCase								=					" " + output_txt
           __lowerCAmelCase								=					tokenizer.encode(lowerCAmelCase__					,						add_special_tokens=lowerCAmelCase__	)
           return output_txt, output_ids


      def  __UpperCAmelCase		(	self	)->	List[str]:
           __lowerCAmelCase								=					self.ta_base_tokenizer
           __lowerCAmelCase								=					tokenizer(["hi</s>", "I went to the gym</s>", "</s>"]	)
           __lowerCAmelCase								=					tokenizer(["hi", "I went to the gym", ""]	)
           self.assertListEqual(batch_with_eos_added["input_ids"]					,						batch_without_eos_added["input_ids"]	)


      def  __UpperCAmelCase		(	self	)->	List[Any]:
           __lowerCAmelCase								=					self.ta_base_tokenizer
           __lowerCAmelCase								=					"Unicode €."
           __lowerCAmelCase								=					tokenizer(lowerCAmelCase__	)
           __lowerCAmelCase								=					[8_8, 1_1_3, 1_0_8, 1_0_2, 1_1_4, 1_0_3, 1_0_4, 3_5, 2_2_9, 1_3_3, 1_7_5, 4_9, 1]
           self.assertEqual(encoded["input_ids"]					,						lowerCAmelCase__	)

           # decoding
           __lowerCAmelCase								=					tokenizer.decode(lowerCAmelCase__	)
           self.assertEqual(lowerCAmelCase__					,						"Unicode €.</s>"	)

           __lowerCAmelCase								=					tokenizer("e è é ê ë"	)
           __lowerCAmelCase								=					[1_0_4, 3_5, 1_9_8, 1_7_1, 3_5, 1_9_8, 1_7_2, 3_5, 1_9_8, 1_7_3, 3_5, 1_9_8, 1_7_4, 1]
           self.assertEqual(encoded["input_ids"]					,						lowerCAmelCase__	)
           # decoding
           __lowerCAmelCase								=					tokenizer.decode(lowerCAmelCase__	)
           self.assertEqual(lowerCAmelCase__					,						"e è é ê ë</s>"	)

           # encode/decode, but with `encode` instead of `__call__`
           self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"	)	)					,						"e è é ê ë</s>"	)


      def  __UpperCAmelCase		(	self	)->	Optional[Any]:
           __lowerCAmelCase								=					self.ta_base_tokenizer
           __lowerCAmelCase								=					["A long paragraph for summarization.", "Another paragraph for summarization."]
           # fmt: off
           __lowerCAmelCase								=					[6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 1, 0]
           # fmt: on
           __lowerCAmelCase								=					tokenizer(lowerCAmelCase__					,						padding=lowerCAmelCase__					,						return_tensors=lowerCAmelCase__	)
           self.assertIsInstance(lowerCAmelCase__					,						lowerCAmelCase__	)

           if FRAMEWORK != "jax":
                __lowerCAmelCase								=					list(batch.input_ids.numpy()[0]	)
           else:
                __lowerCAmelCase								=					list(batch.input_ids.tolist()[0]	)

           self.assertListEqual(lowerCAmelCase__					,						lowerCAmelCase__	)

           self.assertEqual((2, 3_7)					,						batch.input_ids.shape	)
           self.assertEqual((2, 3_7)					,						batch.attention_mask.shape	)


      def  __UpperCAmelCase		(	self	)->	Optional[int]:
           __lowerCAmelCase								=					self.ta_base_tokenizer
           __lowerCAmelCase								=					["A long paragraph for summarization.", "Another paragraph for summarization."]
           __lowerCAmelCase								=					tokenizer(lowerCAmelCase__					,						padding=lowerCAmelCase__					,						return_tensors=lowerCAmelCase__	)
           # check if input_ids are returned and no decoder_input_ids
           self.assertIn("input_ids"					,						lowerCAmelCase__	)
           self.assertIn("attention_mask"					,						lowerCAmelCase__	)
           self.assertNotIn("decoder_input_ids"					,						lowerCAmelCase__	)
           self.assertNotIn("decoder_attention_mask"					,						lowerCAmelCase__	)


      def  __UpperCAmelCase		(	self	)->	List[Any]:
           __lowerCAmelCase								=					self.ta_base_tokenizer
           __lowerCAmelCase								=					[
               "Summary of the text.",
               "Another summary.",
           ]
           __lowerCAmelCase								=					tokenizer(
               text_target=lowerCAmelCase__					,						max_length=3_2					,						padding="max_length"					,						truncation=lowerCAmelCase__					,						return_tensors=lowerCAmelCase__	)
           self.assertEqual(3_2					,						targets["input_ids"].shape[1]	)


      def  __UpperCAmelCase		(	self	)->	List[Any]:
           __lowerCAmelCase								=					self.ta_base_tokenizer
           __lowerCAmelCase								=					["A long paragraph for summarization. </s>"]
           __lowerCAmelCase								=					["Summary of the text. </s>"]
           # fmt: off
           __lowerCAmelCase								=					[6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 3_5, 1]
           __lowerCAmelCase								=					[8_6, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_2_4, 3_5, 1_1_4, 1_0_5, 3_5, 1_1_9, 1_0_7, 1_0_4, 3_5, 1_1_9, 1_0_4, 1_2_3, 1_1_9, 4_9, 3_5, 1]
           # fmt: on

           __lowerCAmelCase								=					tokenizer(lowerCAmelCase__					,						text_target=lowerCAmelCase__	)

           self.assertEqual(lowerCAmelCase__					,						batch["input_ids"][0]	)
           self.assertEqual(lowerCAmelCase__					,						batch["labels"][0]	)


      def  __UpperCAmelCase		(	self	)->	int:
           # safety check on max_len default value so we are sure the test works
           __lowerCAmelCase								=					self.get_tokenizers()
           for tokenizer in tokenizers:
                with self.subTest(F"""{tokenizer.__class__.__name__}"""	):
                     self.assertNotEqual(tokenizer.model_max_length					,						4_2	)

                      # Now let's start the test
           __lowerCAmelCase								=					self.get_tokenizers()
           for tokenizer in tokenizers:
                with self.subTest(F"""{tokenizer.__class__.__name__}"""	):
                     # Isolate this from the other tests because we save additional tokens/etc
                     __lowerCAmelCase								=					tempfile.mkdtemp()

                     __lowerCAmelCase								=					" He is very happy, UNwant\u00E9d,running"
                     __lowerCAmelCase								=					tokenizer.encode(lowerCAmelCase__					,						add_special_tokens=lowerCAmelCase__	)
                     tokenizer.save_pretrained(lowerCAmelCase__	)

                     __lowerCAmelCase								=					tokenizer.__class__.from_pretrained(lowerCAmelCase__	)
                     __lowerCAmelCase								=					after_tokenizer.encode(lowerCAmelCase__					,						add_special_tokens=lowerCAmelCase__	)
                     self.assertListEqual(lowerCAmelCase__					,						lowerCAmelCase__	)

                     shutil.rmtree(lowerCAmelCase__	)

           __lowerCAmelCase								=					self.get_tokenizers(model_max_length=4_2	)
           for tokenizer in tokenizers:
                with self.subTest(F"""{tokenizer.__class__.__name__}"""	):
                     # Isolate this from the other tests because we save additional tokens/etc
                     __lowerCAmelCase								=					tempfile.mkdtemp()

                     __lowerCAmelCase								=					" He is very happy, UNwant\u00E9d,running"
                     tokenizer.add_tokens(["bim", "bambam"]	)
                     __lowerCAmelCase								=					tokenizer.additional_special_tokens
                     additional_special_tokens.append("new_additional_special_token"	)
                     tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}	)
                     __lowerCAmelCase								=					tokenizer.encode(lowerCAmelCase__					,						add_special_tokens=lowerCAmelCase__	)
                     tokenizer.save_pretrained(lowerCAmelCase__	)

                     __lowerCAmelCase								=					tokenizer.__class__.from_pretrained(lowerCAmelCase__	)
                     __lowerCAmelCase								=					after_tokenizer.encode(lowerCAmelCase__					,						add_special_tokens=lowerCAmelCase__	)
                     self.assertListEqual(lowerCAmelCase__					,						lowerCAmelCase__	)
                     self.assertIn("new_additional_special_token"					,						after_tokenizer.additional_special_tokens	)
                     self.assertEqual(after_tokenizer.model_max_length					,						4_2	)

                     __lowerCAmelCase								=					tokenizer.__class__.from_pretrained(lowerCAmelCase__					,						model_max_length=4_3	)
                     self.assertEqual(tokenizer.model_max_length					,						4_3	)

                     shutil.rmtree(lowerCAmelCase__	)


      def  __UpperCAmelCase		(	self	)->	int:
           __lowerCAmelCase								=					[]
           if self.test_slow_tokenizer:
                tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())	)

           if self.test_rust_tokenizer:
                tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())	)

           for tokenizer_class, tokenizer_utils in tokenizer_list:
                with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer_utils.save_pretrained(lowerCAmelCase__	)

                     with open(os.path.join(lowerCAmelCase__					,						"special_tokens_map.json"	)					,						encoding="utf-8"	) as json_file:
                          __lowerCAmelCase								=					json.load(lowerCAmelCase__	)

                     with open(os.path.join(lowerCAmelCase__					,						"tokenizer_config.json"	)					,						encoding="utf-8"	) as json_file:
                          __lowerCAmelCase								=					json.load(lowerCAmelCase__	)

                     __lowerCAmelCase								=					[F"""<extra_id_{i}>""" for i in range(1_2_5	)]

                     __lowerCAmelCase								=					added_tokens_extra_ids + [
                         "an_additional_special_token"
                     ]
                     __lowerCAmelCase								=					added_tokens_extra_ids + [
                         "an_additional_special_token"
                     ]

                     with open(os.path.join(lowerCAmelCase__					,						"special_tokens_map.json"	)					,						"w"					,						encoding="utf-8"	) as outfile:
                          json.dump(lowerCAmelCase__					,						lowerCAmelCase__	)
                     with open(os.path.join(lowerCAmelCase__					,						"tokenizer_config.json"	)					,						"w"					,						encoding="utf-8"	) as outfile:
                          json.dump(lowerCAmelCase__					,						lowerCAmelCase__	)

                     # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                     # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                     # "special_tokens_map.json" files
                     __lowerCAmelCase								=					tokenizer_class.from_pretrained(
                         lowerCAmelCase__					,						)
                     self.assertIn(
                         "an_additional_special_token"					,						tokenizer_without_change_in_init.additional_special_tokens	)
                     # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                     self.assertEqual(
                         ["an_additional_special_token"]					,						tokenizer_without_change_in_init.convert_ids_to_tokens(
                             tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]	)	)					,						)

                     # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                     __lowerCAmelCase								=					added_tokens_extra_ids + [AddedToken("a_new_additional_special_token"					,						lstrip=lowerCAmelCase__	)]
                     __lowerCAmelCase								=					tokenizer_class.from_pretrained(
                         lowerCAmelCase__					,						additional_special_tokens=lowerCAmelCase__					,						)

                     self.assertIn("a_new_additional_special_token"					,						tokenizer.additional_special_tokens	)
                     self.assertEqual(
                         ["a_new_additional_special_token"]					,						tokenizer.convert_ids_to_tokens(
                             tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]	)	)					,						)


      def  __UpperCAmelCase		(	self	)->	Tuple:
           __lowerCAmelCase								=					[]
           if self.test_slow_tokenizer:
                tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())	)

           if self.test_rust_tokenizer:
                tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())	)

           for tokenizer_class, tokenizer_utils in tokenizer_list:
                with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer_utils.save_pretrained(lowerCAmelCase__	)

                     __lowerCAmelCase								=					tokenizer_class.from_pretrained(lowerCAmelCase__	)

                     self.assertTrue(tokenizer.decode([2_5_5]	) == ""	)


      def  __UpperCAmelCase		(	self	)->	Dict:
           pass


      def  __UpperCAmelCase		(	self	)->	Tuple:
           pass


      def  __UpperCAmelCase		(	self	)->	List[str]:
           pass


      def  __UpperCAmelCase		(	self	)->	int:
           pass


      def  __UpperCAmelCase		(	self	)->	int:
           # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
           # and special added tokens as tokens
           __lowerCAmelCase								=					self.get_tokenizers(fast=lowerCAmelCase__					,						do_lower_case=lowerCAmelCase__	)
           for tokenizer in tokenizers:
                with self.subTest(F"""{tokenizer.__class__.__name__}"""	):
                     __lowerCAmelCase								=					["t", "h", "i", "s", " ", "i", "s", " ", "a", " ", "t", "e", "x", "t", "</s>"]
                     __lowerCAmelCase								=					tokenizer.convert_tokens_to_string(lowerCAmelCase__	)

                     self.assertIsInstance(lowerCAmelCase__					,						lowerCAmelCase__	)


      def  __UpperCAmelCase		(	self	)->	List[str]:
           __lowerCAmelCase								=					self.get_tokenizers()
           for tokenizer in tokenizers:
                with self.subTest(F"""{tokenizer.__class__.__name__}"""	):
                     __lowerCAmelCase								=					[
                         "bos_token",
                         "eos_token",
                         "unk_token",
                         "sep_token",
                         "pad_token",
                         "cls_token",
                         "mask_token",
                     ]

                     __lowerCAmelCase								=					0
                     __lowerCAmelCase								=					tokenizer.convert_ids_to_tokens(
                         lowerCAmelCase__					,						skip_special_tokens=lowerCAmelCase__	)

                     for attr in attributes_list:
                          setattr(lowerCAmelCase__					,						attr + "_id"					,						lowerCAmelCase__	)
                          self.assertEqual(getattr(lowerCAmelCase__					,						lowerCAmelCase__	)					,						lowerCAmelCase__	)
                          self.assertEqual(getattr(lowerCAmelCase__					,						attr + "_id"	)					,						lowerCAmelCase__	)

                          setattr(lowerCAmelCase__					,						attr + "_id"					,						lowerCAmelCase__	)
                          self.assertEqual(getattr(lowerCAmelCase__					,						lowerCAmelCase__	)					,						lowerCAmelCase__	)
                          self.assertEqual(getattr(lowerCAmelCase__					,						attr + "_id"	)					,						lowerCAmelCase__	)

                     setattr(lowerCAmelCase__					,						"additional_special_tokens_ids"					,						[]	)
                     self.assertListEqual(getattr(lowerCAmelCase__					,						"additional_special_tokens"	)					,						[]	)
                     self.assertListEqual(getattr(lowerCAmelCase__					,						"additional_special_tokens_ids"	)					,						[]	)

                     setattr(lowerCAmelCase__					,						"additional_special_tokens_ids"					,						[token_id_to_test_setters]	)
                     self.assertListEqual(getattr(lowerCAmelCase__					,						"additional_special_tokens"	)					,						[token_to_test_setters]	)
                     self.assertListEqual(getattr(lowerCAmelCase__					,						"additional_special_tokens_ids"	)					,						[token_id_to_test_setters]	)














lowerCamelCase			:					str         =	'''
# Transformers 설치 방법
! pip install transformers datasets
# 마지막 릴리스 대신 소스에서 설치하려면, 위 명령을 주석으로 바꾸고 아래 명령을 해제하세요.
# ! pip install git+https://github.com/huggingface/transformers.git
'''

lowerCamelCase			:					Tuple         =	[{'''type''': '''code''', '''content''': INSTALL_CONTENT}]
lowerCamelCase			:					Dict         =	{
    '''{processor_class}''': '''FakeProcessorClass''',
    '''{model_class}''': '''FakeModelClass''',
    '''{object_class}''': '''FakeObjectClass''',
}






