




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
     UpperCamelCase								=	'pt'
elif is_tf_available():
     UpperCamelCase								=	'tf'
else:
     UpperCamelCase								=	'jax'





class 					__lowerCamelCase			(       SCREAMING_SNAKE_CASE_      ,							unittest.TestCase       ):




    """simple docstring"""

    snake_case__   = ByTaTokenizer
    snake_case__   = False


    def   a (			self	:	Optional[Any]  ) ->			List[Any]:
         super().setUp()
         lowerCAmelCase__            =					ByTaTokenizer()
         tokenizer.save_pretrained(self.tmpdirname  )


    @cached_property
    def   a (			self	:	str  ) ->			str:
         return ByTaTokenizer.from_pretrained("google/byt5-small"  )


    def   a (			self	:	str  ,      **SCREAMING_SNAKE_CASE__	:	List[str]  ) ->			ByTaTokenizer:
         return self.tokenizer_class.from_pretrained(self.tmpdirname  ,      **__snake_case  )


    def   a (			self	:	Optional[int]  ,      SCREAMING_SNAKE_CASE__	:	List[Any]  ,      SCREAMING_SNAKE_CASE__	:	List[str]=False  ,      SCREAMING_SNAKE_CASE__	:	str=20  ,      SCREAMING_SNAKE_CASE__	:	Dict=5  ) ->			Tuple[str, list]:
         # XXX The default common tokenizer tests assume that every ID is decodable on its own.
         # This assumption is invalid for ByT5 because single bytes might not be
         # valid utf-8 (byte 128 for instance).
         # Here we're overriding the smallest possible method to provide
         # a clean sequence without making the same assumption.

         lowerCAmelCase__            =					[]
         for i in range(len(__snake_case  )  ):
              try:
                   lowerCAmelCase__            =					tokenizer.decode([i]  ,      clean_up_tokenization_spaces=__snake_case  )
              except UnicodeDecodeError:
                   pass
              toks.append((i, tok)  )

         lowerCAmelCase__            =					list(filter(lambda SCREAMING_SNAKE_CASE__  : re.match(r"^[ a-zA-Z]+$"  ,      t[1]  )  ,      __snake_case  )  )
         lowerCAmelCase__            =					list(filter(lambda SCREAMING_SNAKE_CASE__  : [t[0]] == tokenizer.encode(t[1]  ,      add_special_tokens=__snake_case  )  ,      __snake_case  )  )
         if max_length is not None and len(__snake_case  ) > max_length:
              lowerCAmelCase__            =					toks[:max_length]
         if min_length is not None and len(__snake_case  ) < min_length and len(__snake_case  ) > 0:
              while len(__snake_case  ) < min_length:
                   lowerCAmelCase__            =					toks + toks
        # toks_str = [t[1] for t in toks]
         lowerCAmelCase__            =					[t[0] for t in toks]

         # Ensure consistency
         lowerCAmelCase__            =					tokenizer.decode(__snake_case  ,      clean_up_tokenization_spaces=__snake_case  )
         if " " not in output_txt and len(__snake_case  ) > 1:
              lowerCAmelCase__            =					(
                  tokenizer.decode([toks_ids[0]]  ,      clean_up_tokenization_spaces=__snake_case  )
                  + """ """
                  + tokenizer.decode(toks_ids[1:]  ,      clean_up_tokenization_spaces=__snake_case  )
              )
         if with_prefix_space:
              lowerCAmelCase__            =					""" """ + output_txt
         lowerCAmelCase__            =					tokenizer.encode(__snake_case  ,      add_special_tokens=__snake_case  )
         return output_txt, output_ids


    def   a (			self	:	Any  ) ->			Optional[Any]:
         lowerCAmelCase__            =					self.ta_base_tokenizer
         lowerCAmelCase__            =					tokenizer(["hi</s>", "I went to the gym</s>", "</s>"]  )
         lowerCAmelCase__            =					tokenizer(["hi", "I went to the gym", ""]  )
         self.assertListEqual(batch_with_eos_added["input_ids"]  ,      batch_without_eos_added["input_ids"]  )


    def   a (			self	:	List[Any]  ) ->			List[Any]:
         lowerCAmelCase__            =					self.ta_base_tokenizer
         lowerCAmelCase__            =					"""Unicode €."""
         lowerCAmelCase__            =					tokenizer(__snake_case  )
         lowerCAmelCase__            =					[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
         self.assertEqual(encoded["input_ids"]  ,      __snake_case  )

         # decoding
         lowerCAmelCase__            =					tokenizer.decode(__snake_case  )
         self.assertEqual(__snake_case  ,      "Unicode €.</s>"  )

         lowerCAmelCase__            =					tokenizer("e è é ê ë"  )
         lowerCAmelCase__            =					[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
         self.assertEqual(encoded["input_ids"]  ,      __snake_case  )
         # decoding
         lowerCAmelCase__            =					tokenizer.decode(__snake_case  )
         self.assertEqual(__snake_case  ,      "e è é ê ë</s>"  )

         # encode/decode, but with `encode` instead of `__call__`
         self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"  )  )  ,      "e è é ê ë</s>"  )


    def   a (			self	:	str  ) ->			List[Any]:
         lowerCAmelCase__            =					self.ta_base_tokenizer
         lowerCAmelCase__            =					["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
         # fmt: off
         lowerCAmelCase__            =					[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
         # fmt: on
         lowerCAmelCase__            =					tokenizer(__snake_case  ,      padding=__snake_case  ,      return_tensors=__snake_case  )
         self.assertIsInstance(__snake_case  ,      __snake_case  )

         if FRAMEWORK != "jax":
              lowerCAmelCase__            =					list(batch.input_ids.numpy()[0]  )
         else:
              lowerCAmelCase__            =					list(batch.input_ids.tolist()[0]  )

         self.assertListEqual(__snake_case  ,      __snake_case  )

         self.assertEqual((2, 37)  ,      batch.input_ids.shape  )
         self.assertEqual((2, 37)  ,      batch.attention_mask.shape  )


    def   a (			self	:	List[str]  ) ->			int:
         lowerCAmelCase__            =					self.ta_base_tokenizer
         lowerCAmelCase__            =					["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
         lowerCAmelCase__            =					tokenizer(__snake_case  ,      padding=__snake_case  ,      return_tensors=__snake_case  )
         # check if input_ids are returned and no decoder_input_ids
         self.assertIn("input_ids"  ,      __snake_case  )
         self.assertIn("attention_mask"  ,      __snake_case  )
         self.assertNotIn("decoder_input_ids"  ,      __snake_case  )
         self.assertNotIn("decoder_attention_mask"  ,      __snake_case  )


    def   a (			self	:	List[str]  ) ->			List[Any]:
         lowerCAmelCase__            =					self.ta_base_tokenizer
         lowerCAmelCase__            =					[
             """Summary of the text.""",
             """Another summary.""",
         ]
         lowerCAmelCase__            =					tokenizer(
             text_target=__snake_case  ,      max_length=32  ,      padding="max_length"  ,      truncation=__snake_case  ,      return_tensors=__snake_case  )
         self.assertEqual(32  ,      targets["input_ids"].shape[1]  )


    def   a (			self	:	List[str]  ) ->			int:
         lowerCAmelCase__            =					self.ta_base_tokenizer
         lowerCAmelCase__            =					["""A long paragraph for summarization. </s>"""]
         lowerCAmelCase__            =					["""Summary of the text. </s>"""]
         # fmt: off
         lowerCAmelCase__            =					[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
         lowerCAmelCase__            =					[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
         # fmt: on

         lowerCAmelCase__            =					tokenizer(__snake_case  ,      text_target=__snake_case  )

         self.assertEqual(__snake_case  ,      batch["input_ids"][0]  )
         self.assertEqual(__snake_case  ,      batch["labels"][0]  )


    def   a (			self	:	Dict  ) ->			str:
         # safety check on max_len default value so we are sure the test works
         lowerCAmelCase__            =					self.get_tokenizers()
         for tokenizer in tokenizers:
              with self.subTest(f'{tokenizer.__class__.__name__}'  ):
                   self.assertNotEqual(tokenizer.model_max_length  ,      42  )

        # Now let's start the test
         lowerCAmelCase__            =					self.get_tokenizers()
         for tokenizer in tokenizers:
              with self.subTest(f'{tokenizer.__class__.__name__}'  ):
                   # Isolate this from the other tests because we save additional tokens/etc
                   lowerCAmelCase__            =					tempfile.mkdtemp()

                   lowerCAmelCase__            =					""" He is very happy, UNwant\u00E9d,running"""
                   lowerCAmelCase__            =					tokenizer.encode(__snake_case  ,      add_special_tokens=__snake_case  )
                   tokenizer.save_pretrained(__snake_case  )

                   lowerCAmelCase__            =					tokenizer.__class__.from_pretrained(__snake_case  )
                   lowerCAmelCase__            =					after_tokenizer.encode(__snake_case  ,      add_special_tokens=__snake_case  )
                   self.assertListEqual(__snake_case  ,      __snake_case  )

                   shutil.rmtree(__snake_case  )

         lowerCAmelCase__            =					self.get_tokenizers(model_max_length=42  )
         for tokenizer in tokenizers:
              with self.subTest(f'{tokenizer.__class__.__name__}'  ):
                   # Isolate this from the other tests because we save additional tokens/etc
                   lowerCAmelCase__            =					tempfile.mkdtemp()

                   lowerCAmelCase__            =					""" He is very happy, UNwant\u00E9d,running"""
                   tokenizer.add_tokens(["bim", "bambam"]  )
                   lowerCAmelCase__            =					tokenizer.additional_special_tokens
                   additional_special_tokens.append("new_additional_special_token"  )
                   tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}  )
                   lowerCAmelCase__            =					tokenizer.encode(__snake_case  ,      add_special_tokens=__snake_case  )
                   tokenizer.save_pretrained(__snake_case  )

                   lowerCAmelCase__            =					tokenizer.__class__.from_pretrained(__snake_case  )
                   lowerCAmelCase__            =					after_tokenizer.encode(__snake_case  ,      add_special_tokens=__snake_case  )
                   self.assertListEqual(__snake_case  ,      __snake_case  )
                   self.assertIn("new_additional_special_token"  ,      after_tokenizer.additional_special_tokens  )
                   self.assertEqual(after_tokenizer.model_max_length  ,      42  )

                   lowerCAmelCase__            =					tokenizer.__class__.from_pretrained(__snake_case  ,      model_max_length=43  )
                   self.assertEqual(tokenizer.model_max_length  ,      43  )

                   shutil.rmtree(__snake_case  )


    def   a (			self	:	str  ) ->			int:
         lowerCAmelCase__            =					[]
         if self.test_slow_tokenizer:
              tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

         if self.test_rust_tokenizer:
              tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

         for tokenizer_class, tokenizer_utils in tokenizer_list:
              with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer_utils.save_pretrained(__snake_case  )

                   with open(os.path.join(__snake_case  ,      "special_tokens_map.json"  )  ,      encoding="utf-8"  ) as json_file:
                        lowerCAmelCase__            =					json.load(__snake_case  )

                   with open(os.path.join(__snake_case  ,      "tokenizer_config.json"  )  ,      encoding="utf-8"  ) as json_file:
                        lowerCAmelCase__            =					json.load(__snake_case  )

                   lowerCAmelCase__            =					[f'<extra_id_{i}>' for i in range(125  )]

                   lowerCAmelCase__            =					added_tokens_extra_ids + [
                       """an_additional_special_token"""
                   ]
                   lowerCAmelCase__            =					added_tokens_extra_ids + [
                       """an_additional_special_token"""
                   ]

                   with open(os.path.join(__snake_case  ,      "special_tokens_map.json"  )  ,      "w"  ,      encoding="utf-8"  ) as outfile:
                        json.dump(__snake_case  ,      __snake_case  )
                   with open(os.path.join(__snake_case  ,      "tokenizer_config.json"  )  ,      "w"  ,      encoding="utf-8"  ) as outfile:
                        json.dump(__snake_case  ,      __snake_case  )

                   # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                   # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                   # "special_tokens_map.json" files
                   lowerCAmelCase__            =					tokenizer_class.from_pretrained(
                       __snake_case  ,      )
                   self.assertIn(
                       "an_additional_special_token"  ,      tokenizer_without_change_in_init.additional_special_tokens  )
                   # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                   self.assertEqual(
                       ["an_additional_special_token"]  ,      tokenizer_without_change_in_init.convert_ids_to_tokens(
                           tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]  )  )  ,      )

                   # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                   lowerCAmelCase__            =					added_tokens_extra_ids + [AddedToken("a_new_additional_special_token"  ,      lstrip=__snake_case  )]
                   lowerCAmelCase__            =					tokenizer_class.from_pretrained(
                       __snake_case  ,      additional_special_tokens=__snake_case  ,      )

                   self.assertIn("a_new_additional_special_token"  ,      tokenizer.additional_special_tokens  )
                   self.assertEqual(
                       ["a_new_additional_special_token"]  ,      tokenizer.convert_ids_to_tokens(
                           tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]  )  )  ,      )


    def   a (			self	:	Dict  ) ->			Union[str, Any]:
         lowerCAmelCase__            =					[]
         if self.test_slow_tokenizer:
              tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

         if self.test_rust_tokenizer:
              tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

         for tokenizer_class, tokenizer_utils in tokenizer_list:
              with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer_utils.save_pretrained(__snake_case  )

                   lowerCAmelCase__            =					tokenizer_class.from_pretrained(__snake_case  )

                   self.assertTrue(tokenizer.decode([255]  ) == ""  )


    def   a (			self	:	str  ) ->			str:
         pass


    def   a (			self	:	str  ) ->			Tuple:
         pass


    def   a (			self	:	Dict  ) ->			Optional[Any]:
         pass


    def   a (			self	:	Optional[Any]  ) ->			Optional[Any]:
         pass


    def   a (			self	:	List[Any]  ) ->			List[str]:
         # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
         # and special added tokens as tokens
         lowerCAmelCase__            =					self.get_tokenizers(fast=__snake_case  ,      do_lower_case=__snake_case  )
         for tokenizer in tokenizers:
              with self.subTest(f'{tokenizer.__class__.__name__}'  ):
                   lowerCAmelCase__            =					["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
                   lowerCAmelCase__            =					tokenizer.convert_tokens_to_string(__snake_case  )

                   self.assertIsInstance(__snake_case  ,      __snake_case  )


    def   a (			self	:	List[Any]  ) ->			Any:
         lowerCAmelCase__            =					self.get_tokenizers()
         for tokenizer in tokenizers:
              with self.subTest(f'{tokenizer.__class__.__name__}'  ):
                   lowerCAmelCase__            =					[
                       """bos_token""",
                       """eos_token""",
                       """unk_token""",
                       """sep_token""",
                       """pad_token""",
                       """cls_token""",
                       """mask_token""",
                   ]

                   lowerCAmelCase__            =					0
                   lowerCAmelCase__            =					tokenizer.convert_ids_to_tokens(
                       __snake_case  ,      skip_special_tokens=__snake_case  )

                   for attr in attributes_list:
                        setattr(__snake_case  ,      attr + "_id"  ,      __snake_case  )
                        self.assertEqual(getattr(__snake_case  ,      __snake_case  )  ,      __snake_case  )
                        self.assertEqual(getattr(__snake_case  ,      attr + "_id"  )  ,      __snake_case  )

                        setattr(__snake_case  ,      attr + "_id"  ,      __snake_case  )
                        self.assertEqual(getattr(__snake_case  ,      __snake_case  )  ,      __snake_case  )
                        self.assertEqual(getattr(__snake_case  ,      attr + "_id"  )  ,      __snake_case  )

                   setattr(__snake_case  ,      "additional_special_tokens_ids"  ,      []  )
                   self.assertListEqual(getattr(__snake_case  ,      "additional_special_tokens"  )  ,      []  )
                   self.assertListEqual(getattr(__snake_case  ,      "additional_special_tokens_ids"  )  ,      []  )

                   setattr(__snake_case  ,      "additional_special_tokens_ids"  ,      [token_id_to_test_setters]  )
                   self.assertListEqual(getattr(__snake_case  ,      "additional_special_tokens"  )  ,      [token_to_test_setters]  )
                   self.assertListEqual(getattr(__snake_case  ,      "additional_special_tokens_ids"  )  ,      [token_id_to_test_setters]  )








"""simple docstring"""





def   a    (     __UpperCAmelCase      :    list[int]  )    ->							float:
						if not nums:  # Makes sure that the list is not empty
												raise ValueError("""List is empty"""  )

						__magic_name__:	Dict							=       sum(__UpperCAmelCase  ) / len(__UpperCAmelCase  )  # Calculate the average
						return sum(abs(x - average  ) for x in nums  ) / len(__UpperCAmelCase  )


if __name__ == "__main__":
							import doctest

							doctest.testmod()



