





import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
     __A							   =							'pt'
elif is_tf_available():
     __A							   =							'tf'
else:
     __A							   =							'jax'



class   _SCREAMING_SNAKE_CASE		(      _lowerCamelCase       ,				unittest.TestCase       ):







 '''simple docstring'''
 lowercase_						=     ByTaTokenizer
 lowercase_						=     False

 def 				SCREAMING_SNAKE_CASE_   (self							:  List[str])			->Optional[int]:



   '''simple docstring'''



   super().setUp()
   lowerCamelCase__: Optional[Any]         =ByTaTokenizer()
   tokenizer.save_pretrained(self.tmpdirname)

 @cached_property
 def 				SCREAMING_SNAKE_CASE_   (self							:  Optional[Any])			->Optional[Any]:



   '''simple docstring'''



   return ByTaTokenizer.from_pretrained("google/byt5-small")

 def 				SCREAMING_SNAKE_CASE_   (self							:  Optional[Any]			,					**UpperCAmelCase_							:  Union[str, Any])			->ByTaTokenizer:



   '''simple docstring'''



   return self.tokenizer_class.from_pretrained(self.tmpdirname			,					**__UpperCamelCase)

 def 				SCREAMING_SNAKE_CASE_   (self							:  Optional[int]			,					UpperCAmelCase_							:  Tuple			,					UpperCAmelCase_							:  str=False			,					UpperCAmelCase_							:  Optional[Any]=20			,					UpperCAmelCase_							:  str=5)			->Tuple[str, list]:



   '''simple docstring'''



   lowerCamelCase__: Union[str, Any]         =[]
   for i in range(len(__UpperCamelCase)):
     try:
       lowerCamelCase__: Tuple         =tokenizer.decode([i]			,					clean_up_tokenization_spaces=__UpperCamelCase)
     except UnicodeDecodeError:
       pass
     toks.append((i, tok))

   lowerCamelCase__: Optional[int]         =list(filter(lambda UpperCAmelCase_: re.match(R"^[ a-zA-Z]+$"			,					t[1])			,					__UpperCamelCase))
   lowerCamelCase__: Tuple         =list(filter(lambda UpperCAmelCase_: [t[0]] == tokenizer.encode(t[1]			,					add_special_tokens=__UpperCamelCase)			,					__UpperCamelCase))
   if max_length is not None and len(__UpperCamelCase) > max_length:
     lowerCamelCase__: List[str]         =toks[:max_length]
   if min_length is not None and len(__UpperCamelCase) < min_length and len(__UpperCamelCase) > 0:
     while len(__UpperCamelCase) < min_length:
       lowerCamelCase__: Any         =toks + toks
        # toks_str = [t[1] for t in toks]
   lowerCamelCase__: int         =[t[0] for t in toks]

   # Ensure consistency
   lowerCamelCase__: Union[str, Any]         =tokenizer.decode(__UpperCamelCase			,					clean_up_tokenization_spaces=__UpperCamelCase)
   if " " not in output_txt and len(__UpperCamelCase) > 1:
     lowerCamelCase__: Tuple         =(
         tokenizer.decode([toks_ids[0]]			,					clean_up_tokenization_spaces=__UpperCamelCase)
         + " "
         + tokenizer.decode(toks_ids[1:]			,					clean_up_tokenization_spaces=__UpperCamelCase)
     )
   if with_prefix_space:
     lowerCamelCase__: Union[str, Any]         =" " + output_txt
   lowerCamelCase__: int         =tokenizer.encode(__UpperCamelCase			,					add_special_tokens=__UpperCamelCase)
   return output_txt, output_ids

 def 				SCREAMING_SNAKE_CASE_   (self							:  List[str])			->Any:



   '''simple docstring'''



   lowerCamelCase__: int         =self.ta_base_tokenizer
   lowerCamelCase__: int         =tokenizer(["hi</s>", "I went to the gym</s>", "</s>"])
   lowerCamelCase__: Union[str, Any]         =tokenizer(["hi", "I went to the gym", ""])
   self.assertListEqual(batch_with_eos_added["input_ids"]			,					batch_without_eos_added["input_ids"])

 def 				SCREAMING_SNAKE_CASE_   (self							:  Any)			->Optional[Any]:



   '''simple docstring'''



   lowerCamelCase__: Any         =self.ta_base_tokenizer
   lowerCamelCase__: List[str]         ="Unicode €."
   lowerCamelCase__: str         =tokenizer(__UpperCamelCase)
   lowerCamelCase__: Any         =[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
   self.assertEqual(encoded["input_ids"]			,					__UpperCamelCase)

   # decoding
   lowerCamelCase__: str         =tokenizer.decode(__UpperCamelCase)
   self.assertEqual(__UpperCamelCase			,					"Unicode €.</s>")

   lowerCamelCase__: Tuple         =tokenizer("e è é ê ë")
   lowerCamelCase__: Tuple         =[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
   self.assertEqual(encoded["input_ids"]			,					__UpperCamelCase)
   # decoding
   lowerCamelCase__: str         =tokenizer.decode(__UpperCamelCase)
   self.assertEqual(__UpperCamelCase			,					"e è é ê ë</s>")

   # encode/decode, but with `encode` instead of `__call__`
   self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"))			,					"e è é ê ë</s>")

 def 				SCREAMING_SNAKE_CASE_   (self							:  Union[str, Any])			->List[str]:



   '''simple docstring'''



   lowerCamelCase__: Any         =self.ta_base_tokenizer
   lowerCamelCase__: List[Any]         =["A long paragraph for summarization.", "Another paragraph for summarization."]
   # fmt: off
   lowerCamelCase__: Tuple         =[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
   # fmt: on
   lowerCamelCase__: Optional[int]         =tokenizer(__UpperCamelCase			,					padding=__UpperCamelCase			,					return_tensors=__UpperCamelCase)
   self.assertIsInstance(__UpperCamelCase			,					__UpperCamelCase)

   if FRAMEWORK != "jax":
     lowerCamelCase__: int         =list(batch.input_ids.numpy()[0])
   else:
     lowerCamelCase__: Any         =list(batch.input_ids.tolist()[0])

   self.assertListEqual(__UpperCamelCase			,					__UpperCamelCase)

   self.assertEqual((2, 37)			,					batch.input_ids.shape)
   self.assertEqual((2, 37)			,					batch.attention_mask.shape)

 def 				SCREAMING_SNAKE_CASE_   (self							:  List[str])			->List[Any]:



   '''simple docstring'''



   lowerCamelCase__: Optional[int]         =self.ta_base_tokenizer
   lowerCamelCase__: Tuple         =["A long paragraph for summarization.", "Another paragraph for summarization."]
   lowerCamelCase__: int         =tokenizer(__UpperCamelCase			,					padding=__UpperCamelCase			,					return_tensors=__UpperCamelCase)
   # check if input_ids are returned and no decoder_input_ids
   self.assertIn("input_ids"			,					__UpperCamelCase)
   self.assertIn("attention_mask"			,					__UpperCamelCase)
   self.assertNotIn("decoder_input_ids"			,					__UpperCamelCase)
   self.assertNotIn("decoder_attention_mask"			,					__UpperCamelCase)

 def 				SCREAMING_SNAKE_CASE_   (self							:  Dict)			->Optional[Any]:



   '''simple docstring'''



   lowerCamelCase__: str         =self.ta_base_tokenizer
   lowerCamelCase__: Dict         =[
       "Summary of the text.",
       "Another summary.",
   ]
   lowerCamelCase__: List[Any]         =tokenizer(
       text_target=__UpperCamelCase			,					max_length=32			,					padding="max_length"			,					truncation=__UpperCamelCase			,					return_tensors=__UpperCamelCase)
   self.assertEqual(32			,					targets["input_ids"].shape[1])

 def 				SCREAMING_SNAKE_CASE_   (self							:  Union[str, Any])			->Any:



   '''simple docstring'''



   lowerCamelCase__: int         =self.ta_base_tokenizer
   lowerCamelCase__: Optional[Any]         =["A long paragraph for summarization. </s>"]
   lowerCamelCase__: int         =["Summary of the text. </s>"]
   # fmt: off
   lowerCamelCase__: Dict         =[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
   lowerCamelCase__: List[Any]         =[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
   # fmt: on

   lowerCamelCase__: Union[str, Any]         =tokenizer(__UpperCamelCase			,					text_target=__UpperCamelCase)

   self.assertEqual(__UpperCamelCase			,					batch["input_ids"][0])
   self.assertEqual(__UpperCamelCase			,					batch["labels"][0])

 def 				SCREAMING_SNAKE_CASE_   (self							:  Any)			->str:



   '''simple docstring'''



   lowerCamelCase__: Tuple         =self.get_tokenizers()
   for tokenizer in tokenizers:
     with self.subTest(F"""{tokenizer.__class__.__name__}"""):
       self.assertNotEqual(tokenizer.model_max_length			,					42)

        # Now let's start the test
   lowerCamelCase__: Any         =self.get_tokenizers()
   for tokenizer in tokenizers:
     with self.subTest(F"""{tokenizer.__class__.__name__}"""):
       # Isolate this from the other tests because we save additional tokens/etc
       lowerCamelCase__: Union[str, Any]         =tempfile.mkdtemp()

       lowerCamelCase__: Optional[int]         =" He is very happy, UNwant\u00E9d,running"
       lowerCamelCase__: int         =tokenizer.encode(__UpperCamelCase			,					add_special_tokens=__UpperCamelCase)
       tokenizer.save_pretrained(__UpperCamelCase)

       lowerCamelCase__: Tuple         =tokenizer.__class__.from_pretrained(__UpperCamelCase)
       lowerCamelCase__: Optional[Any]         =after_tokenizer.encode(__UpperCamelCase			,					add_special_tokens=__UpperCamelCase)
       self.assertListEqual(__UpperCamelCase			,					__UpperCamelCase)

       shutil.rmtree(__UpperCamelCase)

   lowerCamelCase__: Union[str, Any]         =self.get_tokenizers(model_max_length=42)
   for tokenizer in tokenizers:
     with self.subTest(F"""{tokenizer.__class__.__name__}"""):
       # Isolate this from the other tests because we save additional tokens/etc
       lowerCamelCase__: Any         =tempfile.mkdtemp()

       lowerCamelCase__: Dict         =" He is very happy, UNwant\u00E9d,running"
       tokenizer.add_tokens(["bim", "bambam"])
       lowerCamelCase__: int         =tokenizer.additional_special_tokens
       additional_special_tokens.append("new_additional_special_token")
       tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens})
       lowerCamelCase__: int         =tokenizer.encode(__UpperCamelCase			,					add_special_tokens=__UpperCamelCase)
       tokenizer.save_pretrained(__UpperCamelCase)

       lowerCamelCase__: List[str]         =tokenizer.__class__.from_pretrained(__UpperCamelCase)
       lowerCamelCase__: Dict         =after_tokenizer.encode(__UpperCamelCase			,					add_special_tokens=__UpperCamelCase)
       self.assertListEqual(__UpperCamelCase			,					__UpperCamelCase)
       self.assertIn("new_additional_special_token"			,					after_tokenizer.additional_special_tokens)
       self.assertEqual(after_tokenizer.model_max_length			,					42)

       lowerCamelCase__: Tuple         =tokenizer.__class__.from_pretrained(__UpperCamelCase			,					model_max_length=43)
       self.assertEqual(tokenizer.model_max_length			,					43)

       shutil.rmtree(__UpperCamelCase)

 def 				SCREAMING_SNAKE_CASE_   (self							:  Tuple)			->str:



   '''simple docstring'''



   lowerCamelCase__: Union[str, Any]         =[]
   if self.test_slow_tokenizer:
     tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()))

   if self.test_rust_tokenizer:
     tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()))

   for tokenizer_class, tokenizer_utils in tokenizer_list:
     with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer_utils.save_pretrained(__UpperCamelCase)

       with open(os.path.join(__UpperCamelCase			,					"special_tokens_map.json")			,					encoding="utf-8") as json_file:
         lowerCamelCase__: Optional[Any]         =json.load(__UpperCamelCase)

       with open(os.path.join(__UpperCamelCase			,					"tokenizer_config.json")			,					encoding="utf-8") as json_file:
         lowerCamelCase__: Tuple         =json.load(__UpperCamelCase)

       lowerCamelCase__: List[Any]         =[F"""<extra_id_{i}>""" for i in range(125)]

       lowerCamelCase__: Union[str, Any]         =added_tokens_extra_ids + [
           "an_additional_special_token"
       ]
       lowerCamelCase__: List[str]         =added_tokens_extra_ids + [
           "an_additional_special_token"
       ]

       with open(os.path.join(__UpperCamelCase			,					"special_tokens_map.json")			,					"w"			,					encoding="utf-8") as outfile:
         json.dump(__UpperCamelCase			,					__UpperCamelCase)
       with open(os.path.join(__UpperCamelCase			,					"tokenizer_config.json")			,					"w"			,					encoding="utf-8") as outfile:
         json.dump(__UpperCamelCase			,					__UpperCamelCase)

       # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
       # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
       # "special_tokens_map.json" files
       lowerCamelCase__: List[Any]         =tokenizer_class.from_pretrained(
           __UpperCamelCase			,					)
       self.assertIn(
           "an_additional_special_token"			,					tokenizer_without_change_in_init.additional_special_tokens)
       # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
       self.assertEqual(
           ["an_additional_special_token"]			,					tokenizer_without_change_in_init.convert_ids_to_tokens(
               tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]))			,					)

       # Now we test that we can change the value of additional_special_tokens in the from_pretrained
       lowerCamelCase__: Optional[Any]         =added_tokens_extra_ids + [AddedToken("a_new_additional_special_token"			,					lstrip=__UpperCamelCase)]
       lowerCamelCase__: List[str]         =tokenizer_class.from_pretrained(
           __UpperCamelCase			,					additional_special_tokens=__UpperCamelCase			,					)

       self.assertIn("a_new_additional_special_token"			,					tokenizer.additional_special_tokens)
       self.assertEqual(
           ["a_new_additional_special_token"]			,					tokenizer.convert_ids_to_tokens(
               tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]))			,					)

 def 				SCREAMING_SNAKE_CASE_   (self							:  Optional[Any])			->List[str]:



   '''simple docstring'''



   lowerCamelCase__: Optional[int]         =[]
   if self.test_slow_tokenizer:
     tokenizer_list.append((self.tokenizer_class, self.get_tokenizer()))

   if self.test_rust_tokenizer:
     tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer()))

   for tokenizer_class, tokenizer_utils in tokenizer_list:
     with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer_utils.save_pretrained(__UpperCamelCase)

       lowerCamelCase__: List[Any]         =tokenizer_class.from_pretrained(__UpperCamelCase)

       self.assertTrue(tokenizer.decode([255]) == "")

 def 				SCREAMING_SNAKE_CASE_   (self							:  List[str])			->Any:



   '''simple docstring'''



   pass

 def 				SCREAMING_SNAKE_CASE_   (self							:  Dict)			->Optional[Any]:



   '''simple docstring'''



   pass

 def 				SCREAMING_SNAKE_CASE_   (self							:  Any)			->List[Any]:



   '''simple docstring'''



   pass

 def 				SCREAMING_SNAKE_CASE_   (self							:  Union[str, Any])			->Optional[int]:



   '''simple docstring'''



   pass

 def 				SCREAMING_SNAKE_CASE_   (self							:  int)			->Dict:



   '''simple docstring'''



   lowerCamelCase__: List[str]         =self.get_tokenizers(fast=__UpperCamelCase			,					do_lower_case=__UpperCamelCase)
   for tokenizer in tokenizers:
     with self.subTest(F"""{tokenizer.__class__.__name__}"""):
       lowerCamelCase__: Union[str, Any]         =["t", "h", "i", "s", " ", "i", "s", " ", "a", " ", "t", "e", "x", "t", "</s>"]
       lowerCamelCase__: Dict         =tokenizer.convert_tokens_to_string(__UpperCamelCase)

       self.assertIsInstance(__UpperCamelCase			,					__UpperCamelCase)







 def 				SCREAMING_SNAKE_CASE_   (self							:  Union[str, Any])			->Union[str, Any]:



   '''simple docstring'''



   lowerCamelCase__: Optional[int]         =self.get_tokenizers()
   for tokenizer in tokenizers:
     with self.subTest(F"""{tokenizer.__class__.__name__}"""):
       lowerCamelCase__: str         =[
           "bos_token",
           "eos_token",
           "unk_token",
           "sep_token",
           "pad_token",
           "cls_token",
           "mask_token",
       ]

       lowerCamelCase__: Optional[int]         =0
       lowerCamelCase__: int         =tokenizer.convert_ids_to_tokens(
           __UpperCamelCase			,					skip_special_tokens=__UpperCamelCase)

       for attr in attributes_list:
         setattr(__UpperCamelCase			,					attr + "_id"			,					__UpperCamelCase)
         self.assertEqual(getattr(__UpperCamelCase			,					__UpperCamelCase)			,					__UpperCamelCase)
         self.assertEqual(getattr(__UpperCamelCase			,					attr + "_id")			,					__UpperCamelCase)

         setattr(__UpperCamelCase			,					attr + "_id"			,					__UpperCamelCase)
         self.assertEqual(getattr(__UpperCamelCase			,					__UpperCamelCase)			,					__UpperCamelCase)
         self.assertEqual(getattr(__UpperCamelCase			,					attr + "_id")			,					__UpperCamelCase)

       setattr(__UpperCamelCase			,					"additional_special_tokens_ids"			,					[])
       self.assertListEqual(getattr(__UpperCamelCase			,					"additional_special_tokens")			,					[])
       self.assertListEqual(getattr(__UpperCamelCase			,					"additional_special_tokens_ids")			,					[])

       setattr(__UpperCamelCase			,					"additional_special_tokens_ids"			,					[token_id_to_test_setters])
       self.assertListEqual(getattr(__UpperCamelCase			,					"additional_special_tokens")			,					[token_to_test_setters])
       self.assertListEqual(getattr(__UpperCamelCase			,					"additional_special_tokens_ids")			,					[token_id_to_test_setters])






from typing import List

import numpy as np






def  lowerCamelCase_	(      lowerCAmelCase__	:	dict				)       ->							int:

							'''simple docstring'''






							A      						=    {key: len(lowerCAmelCase__				) for key, value in gen_kwargs.items() if isinstance(lowerCAmelCase__			,	lowerCAmelCase__				)}
							if len(set(lists_lengths.values()				)				) > 1:
														raise RuntimeError(
														    (
														        'Sharding is ambiguous for this dataset: '
														        + 'we found several data sources lists of different lengths, and we don\'t know over which list we should parallelize:\n'
														        + '\n'.join(F'''\t- key {key} has length {length}''' for key, length in lists_lengths.items()				)
														        + '\nTo fix this, check the \'gen_kwargs\' and make sure to use lists only for data sources, '
														        + 'and use tuples otherwise. In the end there should only be one single list, or several lists with the same length.'
														    )				)
							A      						=    max(lists_lengths.values()			,	default=0				)
							return max(1			,	lowerCAmelCase__				)






def  lowerCamelCase_	(      lowerCAmelCase__	:	int			,	lowerCAmelCase__	:	int				)       ->							List[range]:

							'''simple docstring'''






							A      						=    []
							for group_idx in range(lowerCAmelCase__				):
														A      						=    num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs))
														if num_shards_to_add == 0:
																					break
														A      						=    shards_indices_per_group[-1].stop if shards_indices_per_group else 0
														A      						=    range(lowerCAmelCase__			,	start + num_shards_to_add				)
														shards_indices_per_group.append(lowerCAmelCase__				)
							return shards_indices_per_group






def  lowerCamelCase_	(      lowerCAmelCase__	:	dict			,	lowerCAmelCase__	:	int				)       ->							List[dict]:

							'''simple docstring'''






							A      						=    _number_of_shards_in_gen_kwargs(lowerCAmelCase__				)
							if num_shards == 1:
														return [dict(lowerCAmelCase__				)]
							else:
														A      						=    _distribute_shards(num_shards=lowerCAmelCase__			,	max_num_jobs=lowerCAmelCase__				)
														return [
														    {
														        key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]]
														        if isinstance(lowerCAmelCase__			,	lowerCAmelCase__				)
														        else value
														        for key, value in gen_kwargs.items()
														    }
														    for group_idx in range(len(lowerCAmelCase__				)				)
														]






def  lowerCamelCase_	(      lowerCAmelCase__	:	List[dict]				)       ->							dict:

							'''simple docstring'''






							return {
							    key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]]
							    if isinstance(gen_kwargs_list[0][key]			,	lowerCAmelCase__				)
							    else gen_kwargs_list[0][key]
							    for key in gen_kwargs_list[0]
							}






def  lowerCamelCase_	(      lowerCAmelCase__	:	np.random.Generator			,	lowerCAmelCase__	:	dict				)       ->							dict:

							'''simple docstring'''






							A      						=    {len(lowerCAmelCase__				) for value in gen_kwargs.values() if isinstance(lowerCAmelCase__			,	lowerCAmelCase__				)}
							A      						=    {}
							for size in list_sizes:
														A      						=    list(range(lowerCAmelCase__				)				)
														rng.shuffle(indices_per_size[size]				)
							# Now let's copy the gen_kwargs and shuffle the lists based on their sizes
							A      						=    dict(lowerCAmelCase__				)
							for key, value in shuffled_kwargs.items():
														if isinstance(lowerCAmelCase__			,	lowerCAmelCase__				):
																					A      						=    [value[i] for i in indices_per_size[len(lowerCAmelCase__				)]]
							return shuffled_kwargs