




"""simple docstring"""







import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
						__A   :  int					=      'pt'
elif is_tf_available():
						__A   :  int					=      'tf'
else:
						__A   :  List[Any]					=      'jax'




class       lowerCAmelCase__       (			lowerCAmelCase_  ,			unittest.TestCase    ):


				"""simple docstring"""





				__UpperCAmelCase :     Any       =      ByTaTokenizer
				__UpperCAmelCase :     List[Any]       =      False



				def       snake_case    (     self					:	Optional[int]						):
						super().setUp()
						__lowercase				:		Union[str, Any]												=				ByTaTokenizer()
						tokenizer.save_pretrained(self.tmpdirname						)



				@cached_property
				def       snake_case    (     self					:	Tuple						):
						return ByTaTokenizer.from_pretrained("google/byt5-small"						)



				def       snake_case    (     self					:	Union[str, Any]    ,   **lowercase__					:	str						):
						return self.tokenizer_class.from_pretrained(self.tmpdirname    ,   **lowercase__						)



				def       snake_case    (     self					:	int    ,   lowercase__					:	Optional[int]    ,   lowercase__					:	Union[str, Any]=False    ,   lowercase__					:	int=2_0    ,   lowercase__					:	List[Any]=5						):
						# XXX The default common tokenizer tests assume that every ID is decodable on its own.
						# This assumption is invalid for ByT5 because single bytes might not be
						# valid utf-8 (byte 128 for instance).
						# Here we're overriding the smallest possible method to provide
						# a clean sequence without making the same assumption.

						__lowercase				:		int												=				[]
						for i in range(len(lowercase__						)						):
								try:
										__lowercase				:		Union[str, Any]												=				tokenizer.decode([i]    ,   clean_up_tokenization_spaces=lowercase__						)
								except UnicodeDecodeError:
										pass
								toks.append((i, tok)						)

						__lowercase				:		List[str]												=				list(filter(lambda lowercase__						: re.match(r"^[ a-zA-Z]+$"    ,   t[1]						)    ,   lowercase__						)						)
						__lowercase				:		Optional[Any]												=				list(filter(lambda lowercase__						: [t[0]] == tokenizer.encode(t[1]    ,   add_special_tokens=lowercase__						)    ,   lowercase__						)						)
						if max_length is not None and len(lowercase__						) > max_length:
								__lowercase				:		Any												=				toks[:max_length]
						if min_length is not None and len(lowercase__						) < min_length and len(lowercase__						) > 0:
								while len(lowercase__						) < min_length:
										__lowercase				:		Union[str, Any]												=				toks + toks
        # toks_str = [t[1] for t in toks]
						__lowercase				:		str												=				[t[0] for t in toks]

						# Ensure consistency
						__lowercase				:		Optional[Any]												=				tokenizer.decode(lowercase__    ,   clean_up_tokenization_spaces=lowercase__						)
						if " " not in output_txt and len(lowercase__						) > 1:
								__lowercase				:		Any												=				(
								    tokenizer.decode([toks_ids[0]]    ,   clean_up_tokenization_spaces=lowercase__						)
								    + " "
								    + tokenizer.decode(toks_ids[1:]    ,   clean_up_tokenization_spaces=lowercase__						)
								)
						if with_prefix_space:
								__lowercase				:		Dict												=				" " + output_txt
						__lowercase				:		Dict												=				tokenizer.encode(lowercase__    ,   add_special_tokens=lowercase__						)
						return output_txt, output_ids



				def       snake_case    (     self					:	Any						):
						__lowercase				:		int												=				self.ta_base_tokenizer
						__lowercase				:		List[str]												=				tokenizer(["hi</s>", "I went to the gym</s>", "</s>"]						)
						__lowercase				:		Optional[Any]												=				tokenizer(["hi", "I went to the gym", ""]						)
						self.assertListEqual(batch_with_eos_added["input_ids"]    ,   batch_without_eos_added["input_ids"]						)



				def       snake_case    (     self					:	List[str]						):
						__lowercase				:		List[str]												=				self.ta_base_tokenizer
						__lowercase				:		int												=				"Unicode €."
						__lowercase				:		Optional[int]												=				tokenizer(lowercase__						)
						__lowercase				:		List[Any]												=				[8_8, 1_1_3, 1_0_8, 1_0_2, 1_1_4, 1_0_3, 1_0_4, 3_5, 2_2_9, 1_3_3, 1_7_5, 4_9, 1]
						self.assertEqual(encoded["input_ids"]    ,   lowercase__						)

						# decoding
						__lowercase				:		Tuple												=				tokenizer.decode(lowercase__						)
						self.assertEqual(lowercase__    ,   "Unicode €.</s>"						)

						__lowercase				:		Optional[Any]												=				tokenizer("e è é ê ë"						)
						__lowercase				:		str												=				[1_0_4, 3_5, 1_9_8, 1_7_1, 3_5, 1_9_8, 1_7_2, 3_5, 1_9_8, 1_7_3, 3_5, 1_9_8, 1_7_4, 1]
						self.assertEqual(encoded["input_ids"]    ,   lowercase__						)
						# decoding
						__lowercase				:		List[Any]												=				tokenizer.decode(lowercase__						)
						self.assertEqual(lowercase__    ,   "e è é ê ë</s>"						)

						# encode/decode, but with `encode` instead of `__call__`
						self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"						)						)    ,   "e è é ê ë</s>"						)



				def       snake_case    (     self					:	Any						):
						__lowercase				:		str												=				self.ta_base_tokenizer
						__lowercase				:		int												=				["A long paragraph for summarization.", "Another paragraph for summarization."]
						# fmt: off
						__lowercase				:		Any												=				[6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 1, 0]
						# fmt: on
						__lowercase				:		List[Any]												=				tokenizer(lowercase__    ,   padding=lowercase__    ,   return_tensors=lowercase__						)
						self.assertIsInstance(lowercase__    ,   lowercase__						)

						if FRAMEWORK != "jax":
								__lowercase				:		str												=				list(batch.input_ids.numpy()[0]						)
						else:
								__lowercase				:		List[Any]												=				list(batch.input_ids.tolist()[0]						)

						self.assertListEqual(lowercase__    ,   lowercase__						)

						self.assertEqual((2, 3_7)    ,   batch.input_ids.shape						)
						self.assertEqual((2, 3_7)    ,   batch.attention_mask.shape						)



				def       snake_case    (     self					:	Any						):
						__lowercase				:		List[Any]												=				self.ta_base_tokenizer
						__lowercase				:		int												=				["A long paragraph for summarization.", "Another paragraph for summarization."]
						__lowercase				:		Tuple												=				tokenizer(lowercase__    ,   padding=lowercase__    ,   return_tensors=lowercase__						)
						# check if input_ids are returned and no decoder_input_ids
						self.assertIn("input_ids"    ,   lowercase__						)
						self.assertIn("attention_mask"    ,   lowercase__						)
						self.assertNotIn("decoder_input_ids"    ,   lowercase__						)
						self.assertNotIn("decoder_attention_mask"    ,   lowercase__						)



				def       snake_case    (     self					:	Optional[int]						):
						__lowercase				:		Dict												=				self.ta_base_tokenizer
						__lowercase				:		Optional[int]												=				[
						    "Summary of the text.",
						    "Another summary.",
						]
						__lowercase				:		Union[str, Any]												=				tokenizer(
						    text_target=lowercase__    ,   max_length=3_2    ,   padding="max_length"    ,   truncation=lowercase__    ,   return_tensors=lowercase__						)
						self.assertEqual(3_2    ,   targets["input_ids"].shape[1]						)



				def       snake_case    (     self					:	int						):
						__lowercase				:		Any												=				self.ta_base_tokenizer
						__lowercase				:		Dict												=				["A long paragraph for summarization. </s>"]
						__lowercase				:		Optional[int]												=				["Summary of the text. </s>"]
						# fmt: off
						__lowercase				:		str												=				[6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 3_5, 1]
						__lowercase				:		str												=				[8_6, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_2_4, 3_5, 1_1_4, 1_0_5, 3_5, 1_1_9, 1_0_7, 1_0_4, 3_5, 1_1_9, 1_0_4, 1_2_3, 1_1_9, 4_9, 3_5, 1]
						# fmt: on

						__lowercase				:		List[str]												=				tokenizer(lowercase__    ,   text_target=lowercase__						)

						self.assertEqual(lowercase__    ,   batch["input_ids"][0]						)
						self.assertEqual(lowercase__    ,   batch["labels"][0]						)



				def       snake_case    (     self					:	List[str]						):
						# safety check on max_len default value so we are sure the test works
						__lowercase				:		int												=				self.get_tokenizers()
						for tokenizer in tokenizers:
								with self.subTest(f'{tokenizer.__class__.__name__}'						):
										self.assertNotEqual(tokenizer.model_max_length    ,   4_2						)

        # Now let's start the test
						__lowercase				:		int												=				self.get_tokenizers()
						for tokenizer in tokenizers:
								with self.subTest(f'{tokenizer.__class__.__name__}'						):
										# Isolate this from the other tests because we save additional tokens/etc
										__lowercase				:		Dict												=				tempfile.mkdtemp()

										__lowercase				:		List[str]												=				" He is very happy, UNwant\u00E9d,running"
										__lowercase				:		List[str]												=				tokenizer.encode(lowercase__    ,   add_special_tokens=lowercase__						)
										tokenizer.save_pretrained(lowercase__						)

										__lowercase				:		str												=				tokenizer.__class__.from_pretrained(lowercase__						)
										__lowercase				:		Tuple												=				after_tokenizer.encode(lowercase__    ,   add_special_tokens=lowercase__						)
										self.assertListEqual(lowercase__    ,   lowercase__						)

										shutil.rmtree(lowercase__						)

						__lowercase				:		List[str]												=				self.get_tokenizers(model_max_length=4_2						)
						for tokenizer in tokenizers:
								with self.subTest(f'{tokenizer.__class__.__name__}'						):
										# Isolate this from the other tests because we save additional tokens/etc
										__lowercase				:		str												=				tempfile.mkdtemp()

										__lowercase				:		str												=				" He is very happy, UNwant\u00E9d,running"
										tokenizer.add_tokens(["bim", "bambam"]						)
										__lowercase				:		Tuple												=				tokenizer.additional_special_tokens
										additional_special_tokens.append("new_additional_special_token"						)
										tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}						)
										__lowercase				:		Optional[Any]												=				tokenizer.encode(lowercase__    ,   add_special_tokens=lowercase__						)
										tokenizer.save_pretrained(lowercase__						)

										__lowercase				:		Dict												=				tokenizer.__class__.from_pretrained(lowercase__						)
										__lowercase				:		int												=				after_tokenizer.encode(lowercase__    ,   add_special_tokens=lowercase__						)
										self.assertListEqual(lowercase__    ,   lowercase__						)
										self.assertIn("new_additional_special_token"    ,   after_tokenizer.additional_special_tokens						)
										self.assertEqual(after_tokenizer.model_max_length    ,   4_2						)

										__lowercase				:		List[Any]												=				tokenizer.__class__.from_pretrained(lowercase__    ,   model_max_length=4_3						)
										self.assertEqual(tokenizer.model_max_length    ,   4_3						)

										shutil.rmtree(lowercase__						)



				def       snake_case    (     self					:	Any						):
						__lowercase				:		str												=				[]
						if self.test_slow_tokenizer:
								tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

						if self.test_rust_tokenizer:
								tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

						for tokenizer_class, tokenizer_utils in tokenizer_list:
								with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer_utils.save_pretrained(lowercase__						)

										with open(os.path.join(lowercase__    ,   "special_tokens_map.json"						)    ,   encoding="utf-8"						) as json_file:
												__lowercase				:		List[str]												=				json.load(lowercase__						)

										with open(os.path.join(lowercase__    ,   "tokenizer_config.json"						)    ,   encoding="utf-8"						) as json_file:
												__lowercase				:		int												=				json.load(lowercase__						)

										__lowercase				:		List[str]												=				[f'<extra_id_{i}>' for i in range(1_2_5						)]

										__lowercase				:		Dict												=				added_tokens_extra_ids + [
										    "an_additional_special_token"
										]
										__lowercase				:		Optional[int]												=				added_tokens_extra_ids + [
										    "an_additional_special_token"
										]

										with open(os.path.join(lowercase__    ,   "special_tokens_map.json"						)    ,   "w"    ,   encoding="utf-8"						) as outfile:
												json.dump(lowercase__    ,   lowercase__						)
										with open(os.path.join(lowercase__    ,   "tokenizer_config.json"						)    ,   "w"    ,   encoding="utf-8"						) as outfile:
												json.dump(lowercase__    ,   lowercase__						)

										# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
										# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
										# "special_tokens_map.json" files
										__lowercase				:		Union[str, Any]												=				tokenizer_class.from_pretrained(
										    lowercase__    ,   )
										self.assertIn(
										    "an_additional_special_token"    ,   tokenizer_without_change_in_init.additional_special_tokens						)
										# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
										self.assertEqual(
										    ["an_additional_special_token"]    ,   tokenizer_without_change_in_init.convert_ids_to_tokens(
										        tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]						)						)    ,   )

										# Now we test that we can change the value of additional_special_tokens in the from_pretrained
										__lowercase				:		List[Any]												=				added_tokens_extra_ids + [AddedToken("a_new_additional_special_token"    ,   lstrip=lowercase__						)]
										__lowercase				:		Optional[Any]												=				tokenizer_class.from_pretrained(
										    lowercase__    ,   additional_special_tokens=lowercase__    ,   )

										self.assertIn("a_new_additional_special_token"    ,   tokenizer.additional_special_tokens						)
										self.assertEqual(
										    ["a_new_additional_special_token"]    ,   tokenizer.convert_ids_to_tokens(
										        tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]						)						)    ,   )



				def       snake_case    (     self					:	Tuple						):
						__lowercase				:		Union[str, Any]												=				[]
						if self.test_slow_tokenizer:
								tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

						if self.test_rust_tokenizer:
								tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

						for tokenizer_class, tokenizer_utils in tokenizer_list:
								with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer_utils.save_pretrained(lowercase__						)

										__lowercase				:		Dict												=				tokenizer_class.from_pretrained(lowercase__						)

										self.assertTrue(tokenizer.decode([2_5_5]						) == ""						)



				def       snake_case    (     self					:	Optional[Any]						):
						pass



				def       snake_case    (     self					:	Dict						):
						pass



				def       snake_case    (     self					:	Optional[int]						):
						pass



				def       snake_case    (     self					:	Union[str, Any]						):
						pass



				def       snake_case    (     self					:	Union[str, Any]						):
						# The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
						# and special added tokens as tokens
						__lowercase				:		List[Any]												=				self.get_tokenizers(fast=lowercase__    ,   do_lower_case=lowercase__						)
						for tokenizer in tokenizers:
								with self.subTest(f'{tokenizer.__class__.__name__}'						):
										__lowercase				:		Optional[int]												=				["t", "h", "i", "s", " ", "i", "s", " ", "a", " ", "t", "e", "x", "t", "</s>"]
										__lowercase				:		Any												=				tokenizer.convert_tokens_to_string(lowercase__						)

										self.assertIsInstance(lowercase__    ,   lowercase__						)



				def       snake_case    (     self					:	Optional[Any]						):
						__lowercase				:		Tuple												=				self.get_tokenizers()
						for tokenizer in tokenizers:
								with self.subTest(f'{tokenizer.__class__.__name__}'						):
										__lowercase				:		str												=				[
										    "bos_token",
										    "eos_token",
										    "unk_token",
										    "sep_token",
										    "pad_token",
										    "cls_token",
										    "mask_token",
										]

										__lowercase				:		str												=				0
										__lowercase				:		List[Any]												=				tokenizer.convert_ids_to_tokens(
										    lowercase__    ,   skip_special_tokens=lowercase__						)

										for attr in attributes_list:
												setattr(lowercase__    ,   attr + "_id"    ,   lowercase__						)
												self.assertEqual(getattr(lowercase__    ,   lowercase__						)    ,   lowercase__						)
												self.assertEqual(getattr(lowercase__    ,   attr + "_id"						)    ,   lowercase__						)

												setattr(lowercase__    ,   attr + "_id"    ,   lowercase__						)
												self.assertEqual(getattr(lowercase__    ,   lowercase__						)    ,   lowercase__						)
												self.assertEqual(getattr(lowercase__    ,   attr + "_id"						)    ,   lowercase__						)

										setattr(lowercase__    ,   "additional_special_tokens_ids"    ,   []						)
										self.assertListEqual(getattr(lowercase__    ,   "additional_special_tokens"						)    ,   []						)
										self.assertListEqual(getattr(lowercase__    ,   "additional_special_tokens_ids"						)    ,   []						)

										setattr(lowercase__    ,   "additional_special_tokens_ids"    ,   [token_id_to_test_setters]						)
										self.assertListEqual(getattr(lowercase__    ,   "additional_special_tokens"						)    ,   [token_to_test_setters]						)
										self.assertListEqual(getattr(lowercase__    ,   "additional_special_tokens_ids"						)    ,   [token_id_to_test_setters]						)






"""simple docstring"""







from collections import deque

from .hash_table import HashTable




class       lowerCAmelCase__       (			lowerCAmelCase_    ):


				"""simple docstring"""





				def __init__(     self					:	Optional[int]    ,   *lowercase__					:	Any    ,   **lowercase__					:	Union[str, Any]						):
						super().__init__(*lowercase__    ,   **lowercase__						)



				def       snake_case    (     self					:	Any    ,   lowercase__					:	Dict    ,   lowercase__					:	Dict						):
						__lowercase				:		Dict												=				deque([]						) if self.values[key] is None else self.values[key]
						self.values[key].appendleft(lowercase__						)
						__lowercase				:		Union[str, Any]												=				self.values[key]



				def       snake_case    (     self					:	str						):
						return (
						    sum(self.charge_factor - len(lowercase__						) for slot in self.values						)
						    / self.size_table
						    * self.charge_factor
						)



				def       snake_case    (     self					:	Union[str, Any]    ,   lowercase__					:	List[str]    ,   lowercase__					:	str=None						):
						if not (
						    len(self.values[key]						) == self.charge_factor and self.values.count(lowercase__						) == 0
						):
								return key
						return super()._collision_resolution(lowercase__    ,   lowercase__						)
