


'''simple docstring'''






import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
 lowercase				='pt'
elif is_tf_available():
 lowercase				='tf'
else:
 lowercase				='jax'






class    __magic_name__				(						lowerCAmelCase   ,unittest.TestCase							):
       UpperCAmelCase		 =ByTaTokenizer
       UpperCAmelCase		 =False






       def      lowerCAmelCase						(     self)		->    List[str]:




          '''simple docstring'''
          super().setUp()
          _UpperCAmelCase  :  Union[str, Any]				       =ByTaTokenizer()
          tokenizer.save_pretrained(self.tmpdirname)






       @cached_property
       def      lowerCAmelCase						(     self)		->    Optional[Any]:




          '''simple docstring'''
          return ByTaTokenizer.from_pretrained('google/byt5-small')






       def      lowerCAmelCase						(     self    ,				**snake_case)		->    ByTaTokenizer:




          '''simple docstring'''
          return self.tokenizer_class.from_pretrained(self.tmpdirname    ,				**snake_case)






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case=False    ,				snake_case=2_0    ,				snake_case=5)		->    Tuple[str, list]:




          '''simple docstring'''
          # XXX The default common tokenizer tests assume that every ID is decodable on its own.
          # This assumption is invalid for ByT5 because single bytes might not be
          # valid utf-8 (byte 128 for instance).
          # Here we're overriding the smallest possible method to provide
          # a clean sequence without making the same assumption.

          _UpperCAmelCase  :  int				       =[]
          for i in range(len(snake_case)):
             try:
                _UpperCAmelCase  :  List[Any]				       =tokenizer.decode([i]    ,				clean_up_tokenization_spaces=snake_case)
             except UnicodeDecodeError:
                pass
             toks.append((i, tok))

          _UpperCAmelCase  :  Optional[Any]				       =list(filter(lambda snake_case: re.match(r'^[ a-zA-Z]+$'    ,				t[1])    ,				snake_case))
          _UpperCAmelCase  :  Union[str, Any]				       =list(filter(lambda snake_case: [t[0]] == tokenizer.encode(t[1]    ,				add_special_tokens=snake_case)    ,				snake_case))
          if max_length is not None and len(snake_case) > max_length:
             _UpperCAmelCase  :  Tuple				       =toks[:max_length]
          if min_length is not None and len(snake_case) < min_length and len(snake_case) > 0:
             while len(snake_case) < min_length:
                _UpperCAmelCase  :  Dict				       =toks + toks
        # toks_str = [t[1] for t in toks]
          _UpperCAmelCase  :  str				       =[t[0] for t in toks]

          # Ensure consistency
          _UpperCAmelCase  :  int				       =tokenizer.decode(snake_case    ,				clean_up_tokenization_spaces=snake_case)
          if " " not in output_txt and len(snake_case) > 1:
             _UpperCAmelCase  :  Any				       =(
                 tokenizer.decode([toks_ids[0]]    ,				clean_up_tokenization_spaces=snake_case)
                 + ' '
                 + tokenizer.decode(toks_ids[1:]    ,				clean_up_tokenization_spaces=snake_case)
             )
          if with_prefix_space:
             _UpperCAmelCase  :  Optional[int]				       =' ' + output_txt
          _UpperCAmelCase  :  str				       =tokenizer.encode(snake_case    ,				add_special_tokens=snake_case)
          return output_txt, output_ids






       def      lowerCAmelCase						(     self)		->    List[str]:




          '''simple docstring'''
          _UpperCAmelCase  :  int				       =self.ta_base_tokenizer
          _UpperCAmelCase  :  str				       =tokenizer(['hi</s>', 'I went to the gym</s>', '</s>'])
          _UpperCAmelCase  :  List[Any]				       =tokenizer(['hi', 'I went to the gym', ''])
          self.assertListEqual(batch_with_eos_added['input_ids']    ,				batch_without_eos_added['input_ids'])






       def      lowerCAmelCase						(     self)		->    str:




          '''simple docstring'''
          _UpperCAmelCase  :  List[Any]				       =self.ta_base_tokenizer
          _UpperCAmelCase  :  str				       ='Unicode €.'
          _UpperCAmelCase  :  Union[str, Any]				       =tokenizer(snake_case)
          _UpperCAmelCase  :  Any				       =[8_8, 1_1_3, 1_0_8, 1_0_2, 1_1_4, 1_0_3, 1_0_4, 3_5, 2_2_9, 1_3_3, 1_7_5, 4_9, 1]
          self.assertEqual(encoded['input_ids']    ,				snake_case)

          # decoding
          _UpperCAmelCase  :  List[Any]				       =tokenizer.decode(snake_case)
          self.assertEqual(snake_case    ,				'Unicode €.</s>')

          _UpperCAmelCase  :  Any				       =tokenizer('e è é ê ë')
          _UpperCAmelCase  :  Dict				       =[1_0_4, 3_5, 1_9_8, 1_7_1, 3_5, 1_9_8, 1_7_2, 3_5, 1_9_8, 1_7_3, 3_5, 1_9_8, 1_7_4, 1]
          self.assertEqual(encoded['input_ids']    ,				snake_case)
          # decoding
          _UpperCAmelCase  :  Optional[int]				       =tokenizer.decode(snake_case)
          self.assertEqual(snake_case    ,				'e è é ê ë</s>')

          # encode/decode, but with `encode` instead of `__call__`
          self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'))    ,				'e è é ê ë</s>')






       def      lowerCAmelCase						(     self)		->    Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  List[Any]				       =self.ta_base_tokenizer
          _UpperCAmelCase  :  List[str]				       =['A long paragraph for summarization.', 'Another paragraph for summarization.']
          # fmt: off
          _UpperCAmelCase  :  int				       =[6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 1, 0]
          # fmt: on
          _UpperCAmelCase  :  Union[str, Any]				       =tokenizer(snake_case    ,				padding=snake_case    ,				return_tensors=snake_case)
          self.assertIsInstance(snake_case    ,				snake_case)

          if FRAMEWORK != "jax":
             _UpperCAmelCase  :  Any				       =list(batch.input_ids.numpy()[0])
          else:
             _UpperCAmelCase  :  Optional[int]				       =list(batch.input_ids.tolist()[0])

          self.assertListEqual(snake_case    ,				snake_case)

          self.assertEqual((2, 3_7)    ,				batch.input_ids.shape)
          self.assertEqual((2, 3_7)    ,				batch.attention_mask.shape)






       def      lowerCAmelCase						(     self)		->    List[str]:




          '''simple docstring'''
          _UpperCAmelCase  :  List[str]				       =self.ta_base_tokenizer
          _UpperCAmelCase  :  Dict				       =['A long paragraph for summarization.', 'Another paragraph for summarization.']
          _UpperCAmelCase  :  Optional[Any]				       =tokenizer(snake_case    ,				padding=snake_case    ,				return_tensors=snake_case)
          # check if input_ids are returned and no decoder_input_ids
          self.assertIn('input_ids'    ,				snake_case)
          self.assertIn('attention_mask'    ,				snake_case)
          self.assertNotIn('decoder_input_ids'    ,				snake_case)
          self.assertNotIn('decoder_attention_mask'    ,				snake_case)






       def      lowerCAmelCase						(     self)		->    str:




          '''simple docstring'''
          _UpperCAmelCase  :  List[Any]				       =self.ta_base_tokenizer
          _UpperCAmelCase  :  List[Any]				       =[
              'Summary of the text.',
              'Another summary.',
          ]
          _UpperCAmelCase  :  str				       =tokenizer(
              text_target=snake_case    ,				max_length=3_2    ,				padding='max_length'    ,				truncation=snake_case    ,				return_tensors=snake_case)
          self.assertEqual(3_2    ,				targets['input_ids'].shape[1])






       def      lowerCAmelCase						(     self)		->    int:




          '''simple docstring'''
          _UpperCAmelCase  :  int				       =self.ta_base_tokenizer
          _UpperCAmelCase  :  str				       =['A long paragraph for summarization. </s>']
          _UpperCAmelCase  :  List[Any]				       =['Summary of the text. </s>']
          # fmt: off
          _UpperCAmelCase  :  Any				       =[6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 3_5, 1]
          _UpperCAmelCase  :  Dict				       =[8_6, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_2_4, 3_5, 1_1_4, 1_0_5, 3_5, 1_1_9, 1_0_7, 1_0_4, 3_5, 1_1_9, 1_0_4, 1_2_3, 1_1_9, 4_9, 3_5, 1]
          # fmt: on

          _UpperCAmelCase  :  Optional[int]				       =tokenizer(snake_case    ,				text_target=snake_case)

          self.assertEqual(snake_case    ,				batch['input_ids'][0])
          self.assertEqual(snake_case    ,				batch['labels'][0])






       def      lowerCAmelCase						(     self)		->    int:




          '''simple docstring'''
          # safety check on max_len default value so we are sure the test works
          _UpperCAmelCase  :  int				       =self.get_tokenizers()
          for tokenizer in tokenizers:
             with self.subTest(f"{tokenizer.__class__.__name__}"):
                self.assertNotEqual(tokenizer.model_max_length    ,				4_2)

        # Now let's start the test
          _UpperCAmelCase  :  Dict				       =self.get_tokenizers()
          for tokenizer in tokenizers:
             with self.subTest(f"{tokenizer.__class__.__name__}"):
                # Isolate this from the other tests because we save additional tokens/etc
                _UpperCAmelCase  :  List[str]				       =tempfile.mkdtemp()

                _UpperCAmelCase  :  Optional[Any]				       =' He is very happy, UNwant\u00E9d,running'
                _UpperCAmelCase  :  str				       =tokenizer.encode(snake_case    ,				add_special_tokens=snake_case)
                tokenizer.save_pretrained(snake_case)

                _UpperCAmelCase  :  str				       =tokenizer.__class__.from_pretrained(snake_case)
                _UpperCAmelCase  :  Optional[int]				       =after_tokenizer.encode(snake_case    ,				add_special_tokens=snake_case)
                self.assertListEqual(snake_case    ,				snake_case)

                shutil.rmtree(snake_case)

          _UpperCAmelCase  :  Tuple				       =self.get_tokenizers(model_max_length=4_2)
          for tokenizer in tokenizers:
             with self.subTest(f"{tokenizer.__class__.__name__}"):
                # Isolate this from the other tests because we save additional tokens/etc
                _UpperCAmelCase  :  int				       =tempfile.mkdtemp()

                _UpperCAmelCase  :  Any				       =' He is very happy, UNwant\u00E9d,running'
                tokenizer.add_tokens(['bim', 'bambam'])
                _UpperCAmelCase  :  Tuple				       =tokenizer.additional_special_tokens
                additional_special_tokens.append('new_additional_special_token')
                tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens})
                _UpperCAmelCase  :  List[Any]				       =tokenizer.encode(snake_case    ,				add_special_tokens=snake_case)
                tokenizer.save_pretrained(snake_case)

                _UpperCAmelCase  :  List[str]				       =tokenizer.__class__.from_pretrained(snake_case)
                _UpperCAmelCase  :  Optional[int]				       =after_tokenizer.encode(snake_case    ,				add_special_tokens=snake_case)
                self.assertListEqual(snake_case    ,				snake_case)
                self.assertIn('new_additional_special_token'    ,				after_tokenizer.additional_special_tokens)
                self.assertEqual(after_tokenizer.model_max_length    ,				4_2)

                _UpperCAmelCase  :  List[str]				       =tokenizer.__class__.from_pretrained(snake_case    ,				model_max_length=4_3)
                self.assertEqual(tokenizer.model_max_length    ,				4_3)

                shutil.rmtree(snake_case)






       def      lowerCAmelCase						(     self)		->    Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase  :  Dict				       =[]
          if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()))

          if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()))

          for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                tokenizer_utils.save_pretrained(snake_case)

                with open(os.path.join(snake_case    ,				'special_tokens_map.json')    ,				encoding='utf-8') as json_file:
                   _UpperCAmelCase  :  List[Any]				       =json.load(snake_case)

                with open(os.path.join(snake_case    ,				'tokenizer_config.json')    ,				encoding='utf-8') as json_file:
                   _UpperCAmelCase  :  Any				       =json.load(snake_case)

                _UpperCAmelCase  :  List[Any]				       =[f"<extra_id_{i}>" for i in range(1_2_5)]

                _UpperCAmelCase  :  Any				       =added_tokens_extra_ids + [
                    'an_additional_special_token'
                ]
                _UpperCAmelCase  :  Tuple				       =added_tokens_extra_ids + [
                    'an_additional_special_token'
                ]

                with open(os.path.join(snake_case    ,				'special_tokens_map.json')    ,				'w'    ,				encoding='utf-8') as outfile:
                   json.dump(snake_case    ,				snake_case)
                with open(os.path.join(snake_case    ,				'tokenizer_config.json')    ,				'w'    ,				encoding='utf-8') as outfile:
                   json.dump(snake_case    ,				snake_case)

                # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                # "special_tokens_map.json" files
                _UpperCAmelCase  :  Union[str, Any]				       =tokenizer_class.from_pretrained(
                    snake_case    ,				)
                self.assertIn(
                    'an_additional_special_token'    ,				tokenizer_without_change_in_init.additional_special_tokens)
                # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                self.assertEqual(
                    ['an_additional_special_token']    ,				tokenizer_without_change_in_init.convert_ids_to_tokens(
                        tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']))    ,				)

                # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                _UpperCAmelCase  :  str				       =added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'    ,				lstrip=snake_case)]
                _UpperCAmelCase  :  List[Any]				       =tokenizer_class.from_pretrained(
                    snake_case    ,				additional_special_tokens=snake_case    ,				)

                self.assertIn('a_new_additional_special_token'    ,				tokenizer.additional_special_tokens)
                self.assertEqual(
                    ['a_new_additional_special_token']    ,				tokenizer.convert_ids_to_tokens(
                        tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']))    ,				)






       def      lowerCAmelCase						(     self)		->    List[str]:




          '''simple docstring'''
          _UpperCAmelCase  :  Tuple				       =[]
          if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()))

          if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()))

          for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                tokenizer_utils.save_pretrained(snake_case)

                _UpperCAmelCase  :  int				       =tokenizer_class.from_pretrained(snake_case)

                self.assertTrue(tokenizer.decode([2_5_5]) == '')






       def      lowerCAmelCase						(     self)		->    int:




          '''simple docstring'''
          pass






       def      lowerCAmelCase						(     self)		->    Dict:




          '''simple docstring'''
          pass






       def      lowerCAmelCase						(     self)		->    str:




          '''simple docstring'''
          pass






       def      lowerCAmelCase						(     self)		->    Optional[int]:




          '''simple docstring'''
          pass






       def      lowerCAmelCase						(     self)		->    List[str]:




          '''simple docstring'''
          # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
          # and special added tokens as tokens
          _UpperCAmelCase  :  Dict				       =self.get_tokenizers(fast=snake_case    ,				do_lower_case=snake_case)
          for tokenizer in tokenizers:
             with self.subTest(f"{tokenizer.__class__.__name__}"):
                _UpperCAmelCase  :  str				       =['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
                _UpperCAmelCase  :  List[str]				       =tokenizer.convert_tokens_to_string(snake_case)

                self.assertIsInstance(snake_case    ,				snake_case)






       def      lowerCAmelCase						(     self)		->    List[str]:




          '''simple docstring'''
          _UpperCAmelCase  :  Tuple				       =self.get_tokenizers()
          for tokenizer in tokenizers:
             with self.subTest(f"{tokenizer.__class__.__name__}"):
                _UpperCAmelCase  :  List[str]				       =[
                    'bos_token',
                    'eos_token',
                    'unk_token',
                    'sep_token',
                    'pad_token',
                    'cls_token',
                    'mask_token',
                ]

                _UpperCAmelCase  :  List[str]				       =0
                _UpperCAmelCase  :  Union[str, Any]				       =tokenizer.convert_ids_to_tokens(
                    snake_case    ,				skip_special_tokens=snake_case)

                for attr in attributes_list:
                   setattr(snake_case    ,				attr + '_id'    ,				snake_case)
                   self.assertEqual(getattr(snake_case    ,				snake_case)    ,				snake_case)
                   self.assertEqual(getattr(snake_case    ,				attr + '_id')    ,				snake_case)

                   setattr(snake_case    ,				attr + '_id'    ,				snake_case)
                   self.assertEqual(getattr(snake_case    ,				snake_case)    ,				snake_case)
                   self.assertEqual(getattr(snake_case    ,				attr + '_id')    ,				snake_case)

                setattr(snake_case    ,				'additional_special_tokens_ids'    ,				[])
                self.assertListEqual(getattr(snake_case    ,				'additional_special_tokens')    ,				[])
                self.assertListEqual(getattr(snake_case    ,				'additional_special_tokens_ids')    ,				[])

                setattr(snake_case    ,				'additional_special_tokens_ids'    ,				[token_id_to_test_setters])
                self.assertListEqual(getattr(snake_case    ,				'additional_special_tokens')    ,				[token_to_test_setters])
                self.assertListEqual(getattr(snake_case    ,				'additional_special_tokens_ids')    ,				[token_id_to_test_setters])





'''simple docstring'''






import warnings
from functools import wraps
from typing import Callable





def lowerCamelCase__     (		__lowerCamelCase			:				Callable				):







   '''simple docstring'''




   @wraps(__lowerCamelCase				)
   def _inner_fn(*__lowerCamelCase			:				int			,       **__lowerCamelCase			:				Optional[Any]				):
      warnings.warn(
          (f"'{fn.__name__}' is experimental and might be subject to breaking changes in the future.")			,       __lowerCamelCase			,       )
      return fn(*__lowerCamelCase			,       **__lowerCamelCase				)

   return _inner_fn

