




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
				a													=			"pt"
elif is_tf_available():
				a													=			"tf"
else:
				a													=			"jax"




class       _A				(		__lowercase       ,     unittest.TestCase     ):
		__a					    =       ByTaTokenizer
		__a					    =       False






		def 				UpperCAmelCase (    self			):
								super().setUp()
								_UpperCAmelCase      =			ByTaTokenizer()
								tokenizer.save_pretrained(self.tmpdirname			)






		@cached_property
		def 				UpperCAmelCase (    self			):
								return ByTaTokenizer.from_pretrained("""google/byt5-small"""			)






		def 				UpperCAmelCase (    self						,		**_SCREAMING_SNAKE_CASE			):
								return self.tokenizer_class.from_pretrained(self.tmpdirname						,		**_SCREAMING_SNAKE_CASE			)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE=False						,		_SCREAMING_SNAKE_CASE=20						,		_SCREAMING_SNAKE_CASE=5			):
								# XXX The default common tokenizer tests assume that every ID is decodable on its own.
								# This assumption is invalid for ByT5 because single bytes might not be
								# valid utf-8 (byte 128 for instance).
								# Here we're overriding the smallest possible method to provide
								# a clean sequence without making the same assumption.

								_UpperCAmelCase      =			[]
								for i in range(len(_SCREAMING_SNAKE_CASE			)			):
														try:
																				_UpperCAmelCase      =			tokenizer.decode([i]						,		clean_up_tokenization_spaces=_SCREAMING_SNAKE_CASE			)
														except UnicodeDecodeError:
																				pass
														toks.append((i, tok)			)

								_UpperCAmelCase      =			list(filter(lambda _SCREAMING_SNAKE_CASE			: re.match(r"""^[ a-zA-Z]+$"""						,		t[1]			)						,		_SCREAMING_SNAKE_CASE			)			)
								_UpperCAmelCase      =			list(filter(lambda _SCREAMING_SNAKE_CASE			: [t[0]] == tokenizer.encode(t[1]						,		add_special_tokens=_SCREAMING_SNAKE_CASE			)						,		_SCREAMING_SNAKE_CASE			)			)
								if max_length is not None and len(_SCREAMING_SNAKE_CASE			) > max_length:
														_UpperCAmelCase      =			toks[:max_length]
								if min_length is not None and len(_SCREAMING_SNAKE_CASE			) < min_length and len(_SCREAMING_SNAKE_CASE			) > 0:
														while len(_SCREAMING_SNAKE_CASE			) < min_length:
																				_UpperCAmelCase      =			toks + toks
        # toks_str = [t[1] for t in toks]
								_UpperCAmelCase      =			[t[0] for t in toks]

								# Ensure consistency
								_UpperCAmelCase      =			tokenizer.decode(_SCREAMING_SNAKE_CASE						,		clean_up_tokenization_spaces=_SCREAMING_SNAKE_CASE			)
								if " " not in output_txt and len(_SCREAMING_SNAKE_CASE			) > 1:
														_UpperCAmelCase      =			(
														    tokenizer.decode([toks_ids[0]]						,		clean_up_tokenization_spaces=_SCREAMING_SNAKE_CASE			)
														    + """ """
														    + tokenizer.decode(toks_ids[1:]						,		clean_up_tokenization_spaces=_SCREAMING_SNAKE_CASE			)
														)
								if with_prefix_space:
														_UpperCAmelCase      =			""" """ + output_txt
								_UpperCAmelCase      =			tokenizer.encode(_SCREAMING_SNAKE_CASE						,		add_special_tokens=_SCREAMING_SNAKE_CASE			)
								return output_txt, output_ids






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			self.ta_base_tokenizer
								_UpperCAmelCase      =			tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]			)
								_UpperCAmelCase      =			tokenizer(["""hi""", """I went to the gym""", """"""]			)
								self.assertListEqual(batch_with_eos_added["""input_ids"""]						,		batch_without_eos_added["""input_ids"""]			)






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			self.ta_base_tokenizer
								_UpperCAmelCase      =			"""Unicode €."""
								_UpperCAmelCase      =			tokenizer(_SCREAMING_SNAKE_CASE			)
								_UpperCAmelCase      =			[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
								self.assertEqual(encoded["""input_ids"""]						,		_SCREAMING_SNAKE_CASE			)

								# decoding
								_UpperCAmelCase      =			tokenizer.decode(_SCREAMING_SNAKE_CASE			)
								self.assertEqual(_SCREAMING_SNAKE_CASE						,		"""Unicode €.</s>"""			)

								_UpperCAmelCase      =			tokenizer("""e è é ê ë"""			)
								_UpperCAmelCase      =			[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
								self.assertEqual(encoded["""input_ids"""]						,		_SCREAMING_SNAKE_CASE			)
								# decoding
								_UpperCAmelCase      =			tokenizer.decode(_SCREAMING_SNAKE_CASE			)
								self.assertEqual(_SCREAMING_SNAKE_CASE						,		"""e è é ê ë</s>"""			)

								# encode/decode, but with `encode` instead of `__call__`
								self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""			)			)						,		"""e è é ê ë</s>"""			)






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			self.ta_base_tokenizer
								_UpperCAmelCase      =			["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
								# fmt: off
								_UpperCAmelCase      =			[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
								# fmt: on
								_UpperCAmelCase      =			tokenizer(_SCREAMING_SNAKE_CASE						,		padding=_SCREAMING_SNAKE_CASE						,		return_tensors=_SCREAMING_SNAKE_CASE			)
								self.assertIsInstance(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			)

								if FRAMEWORK != "jax":
														_UpperCAmelCase      =			list(batch.input_ids.numpy()[0]			)
								else:
														_UpperCAmelCase      =			list(batch.input_ids.tolist()[0]			)

								self.assertListEqual(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			)

								self.assertEqual((2, 37)						,		batch.input_ids.shape			)
								self.assertEqual((2, 37)						,		batch.attention_mask.shape			)






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			self.ta_base_tokenizer
								_UpperCAmelCase      =			["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
								_UpperCAmelCase      =			tokenizer(_SCREAMING_SNAKE_CASE						,		padding=_SCREAMING_SNAKE_CASE						,		return_tensors=_SCREAMING_SNAKE_CASE			)
								# check if input_ids are returned and no decoder_input_ids
								self.assertIn("""input_ids"""						,		_SCREAMING_SNAKE_CASE			)
								self.assertIn("""attention_mask"""						,		_SCREAMING_SNAKE_CASE			)
								self.assertNotIn("""decoder_input_ids"""						,		_SCREAMING_SNAKE_CASE			)
								self.assertNotIn("""decoder_attention_mask"""						,		_SCREAMING_SNAKE_CASE			)






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			self.ta_base_tokenizer
								_UpperCAmelCase      =			[
								    """Summary of the text.""",
								    """Another summary.""",
								]
								_UpperCAmelCase      =			tokenizer(
								    text_target=_SCREAMING_SNAKE_CASE						,		max_length=32						,		padding="""max_length"""						,		truncation=_SCREAMING_SNAKE_CASE						,		return_tensors=_SCREAMING_SNAKE_CASE			)
								self.assertEqual(32						,		targets["""input_ids"""].shape[1]			)






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			self.ta_base_tokenizer
								_UpperCAmelCase      =			["""A long paragraph for summarization. </s>"""]
								_UpperCAmelCase      =			["""Summary of the text. </s>"""]
								# fmt: off
								_UpperCAmelCase      =			[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
								_UpperCAmelCase      =			[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
								# fmt: on

								_UpperCAmelCase      =			tokenizer(_SCREAMING_SNAKE_CASE						,		text_target=_SCREAMING_SNAKE_CASE			)

								self.assertEqual(_SCREAMING_SNAKE_CASE						,		batch["""input_ids"""][0]			)
								self.assertEqual(_SCREAMING_SNAKE_CASE						,		batch["""labels"""][0]			)






		def 				UpperCAmelCase (    self			):
								# safety check on max_len default value so we are sure the test works
								_UpperCAmelCase      =			self.get_tokenizers()
								for tokenizer in tokenizers:
														with self.subTest(F"{tokenizer.__class__.__name__}"			):
																				self.assertNotEqual(tokenizer.model_max_length						,		42			)

        # Now let's start the test
								_UpperCAmelCase      =			self.get_tokenizers()
								for tokenizer in tokenizers:
														with self.subTest(F"{tokenizer.__class__.__name__}"			):
																				# Isolate this from the other tests because we save additional tokens/etc
																				_UpperCAmelCase      =			tempfile.mkdtemp()

																				_UpperCAmelCase      =			""" He is very happy, UNwant\u00E9d,running"""
																				_UpperCAmelCase      =			tokenizer.encode(_SCREAMING_SNAKE_CASE						,		add_special_tokens=_SCREAMING_SNAKE_CASE			)
																				tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE			)

																				_UpperCAmelCase      =			tokenizer.__class__.from_pretrained(_SCREAMING_SNAKE_CASE			)
																				_UpperCAmelCase      =			after_tokenizer.encode(_SCREAMING_SNAKE_CASE						,		add_special_tokens=_SCREAMING_SNAKE_CASE			)
																				self.assertListEqual(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			)

																				shutil.rmtree(_SCREAMING_SNAKE_CASE			)

								_UpperCAmelCase      =			self.get_tokenizers(model_max_length=42			)
								for tokenizer in tokenizers:
														with self.subTest(F"{tokenizer.__class__.__name__}"			):
																				# Isolate this from the other tests because we save additional tokens/etc
																				_UpperCAmelCase      =			tempfile.mkdtemp()

																				_UpperCAmelCase      =			""" He is very happy, UNwant\u00E9d,running"""
																				tokenizer.add_tokens(["""bim""", """bambam"""]			)
																				_UpperCAmelCase      =			tokenizer.additional_special_tokens
																				additional_special_tokens.append("""new_additional_special_token"""			)
																				tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}			)
																				_UpperCAmelCase      =			tokenizer.encode(_SCREAMING_SNAKE_CASE						,		add_special_tokens=_SCREAMING_SNAKE_CASE			)
																				tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE			)

																				_UpperCAmelCase      =			tokenizer.__class__.from_pretrained(_SCREAMING_SNAKE_CASE			)
																				_UpperCAmelCase      =			after_tokenizer.encode(_SCREAMING_SNAKE_CASE						,		add_special_tokens=_SCREAMING_SNAKE_CASE			)
																				self.assertListEqual(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			)
																				self.assertIn("""new_additional_special_token"""						,		after_tokenizer.additional_special_tokens			)
																				self.assertEqual(after_tokenizer.model_max_length						,		42			)

																				_UpperCAmelCase      =			tokenizer.__class__.from_pretrained(_SCREAMING_SNAKE_CASE						,		model_max_length=43			)
																				self.assertEqual(tokenizer.model_max_length						,		43			)

																				shutil.rmtree(_SCREAMING_SNAKE_CASE			)






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			[]
								if self.test_slow_tokenizer:
														tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())			)

								if self.test_rust_tokenizer:
														tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())			)

								for tokenizer_class, tokenizer_utils in tokenizer_list:
														with tempfile.TemporaryDirectory() as tmp_dir:
																				tokenizer_utils.save_pretrained(_SCREAMING_SNAKE_CASE			)

																				with open(os.path.join(_SCREAMING_SNAKE_CASE						,		"""special_tokens_map.json"""			)						,		encoding="""utf-8"""			) as json_file:
																										_UpperCAmelCase      =			json.load(_SCREAMING_SNAKE_CASE			)

																				with open(os.path.join(_SCREAMING_SNAKE_CASE						,		"""tokenizer_config.json"""			)						,		encoding="""utf-8"""			) as json_file:
																										_UpperCAmelCase      =			json.load(_SCREAMING_SNAKE_CASE			)

																				_UpperCAmelCase      =			[F"<extra_id_{i}>" for i in range(125			)]

																				_UpperCAmelCase      =			added_tokens_extra_ids + [
																				    """an_additional_special_token"""
																				]
																				_UpperCAmelCase      =			added_tokens_extra_ids + [
																				    """an_additional_special_token"""
																				]

																				with open(os.path.join(_SCREAMING_SNAKE_CASE						,		"""special_tokens_map.json"""			)						,		"""w"""						,		encoding="""utf-8"""			) as outfile:
																										json.dump(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			)
																				with open(os.path.join(_SCREAMING_SNAKE_CASE						,		"""tokenizer_config.json"""			)						,		"""w"""						,		encoding="""utf-8"""			) as outfile:
																										json.dump(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			)

																				# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																				# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																				# "special_tokens_map.json" files
																				_UpperCAmelCase      =			tokenizer_class.from_pretrained(
																				    _SCREAMING_SNAKE_CASE						,		)
																				self.assertIn(
																				    """an_additional_special_token"""						,		tokenizer_without_change_in_init.additional_special_tokens			)
																				# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																				self.assertEqual(
																				    ["""an_additional_special_token"""]						,		tokenizer_without_change_in_init.convert_ids_to_tokens(
																				        tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]			)			)						,		)

																				# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																				_UpperCAmelCase      =			added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""						,		lstrip=_SCREAMING_SNAKE_CASE			)]
																				_UpperCAmelCase      =			tokenizer_class.from_pretrained(
																				    _SCREAMING_SNAKE_CASE						,		additional_special_tokens=_SCREAMING_SNAKE_CASE						,		)

																				self.assertIn("""a_new_additional_special_token"""						,		tokenizer.additional_special_tokens			)
																				self.assertEqual(
																				    ["""a_new_additional_special_token"""]						,		tokenizer.convert_ids_to_tokens(
																				        tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]			)			)						,		)






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			[]
								if self.test_slow_tokenizer:
														tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())			)

								if self.test_rust_tokenizer:
														tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())			)

								for tokenizer_class, tokenizer_utils in tokenizer_list:
														with tempfile.TemporaryDirectory() as tmp_dir:
																				tokenizer_utils.save_pretrained(_SCREAMING_SNAKE_CASE			)

																				_UpperCAmelCase      =			tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE			)

																				self.assertTrue(tokenizer.decode([255]			) == """"""			)






		def 				UpperCAmelCase (    self			):
								pass






		def 				UpperCAmelCase (    self			):
								pass






		def 				UpperCAmelCase (    self			):
								pass






		def 				UpperCAmelCase (    self			):
								pass






		def 				UpperCAmelCase (    self			):
								# The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
								# and special added tokens as tokens
								_UpperCAmelCase      =			self.get_tokenizers(fast=_SCREAMING_SNAKE_CASE						,		do_lower_case=_SCREAMING_SNAKE_CASE			)
								for tokenizer in tokenizers:
														with self.subTest(F"{tokenizer.__class__.__name__}"			):
																				_UpperCAmelCase      =			["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
																				_UpperCAmelCase      =			tokenizer.convert_tokens_to_string(_SCREAMING_SNAKE_CASE			)

																				self.assertIsInstance(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			)






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			self.get_tokenizers()
								for tokenizer in tokenizers:
														with self.subTest(F"{tokenizer.__class__.__name__}"			):
																				_UpperCAmelCase      =			[
																				    """bos_token""",
																				    """eos_token""",
																				    """unk_token""",
																				    """sep_token""",
																				    """pad_token""",
																				    """cls_token""",
																				    """mask_token""",
																				]

																				_UpperCAmelCase      =			0
																				_UpperCAmelCase      =			tokenizer.convert_ids_to_tokens(
																				    _SCREAMING_SNAKE_CASE						,		skip_special_tokens=_SCREAMING_SNAKE_CASE			)

																				for attr in attributes_list:
																										setattr(_SCREAMING_SNAKE_CASE						,		attr + """_id"""						,		_SCREAMING_SNAKE_CASE			)
																										self.assertEqual(getattr(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			)						,		_SCREAMING_SNAKE_CASE			)
																										self.assertEqual(getattr(_SCREAMING_SNAKE_CASE						,		attr + """_id"""			)						,		_SCREAMING_SNAKE_CASE			)

																										setattr(_SCREAMING_SNAKE_CASE						,		attr + """_id"""						,		_SCREAMING_SNAKE_CASE			)
																										self.assertEqual(getattr(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			)						,		_SCREAMING_SNAKE_CASE			)
																										self.assertEqual(getattr(_SCREAMING_SNAKE_CASE						,		attr + """_id"""			)						,		_SCREAMING_SNAKE_CASE			)

																				setattr(_SCREAMING_SNAKE_CASE						,		"""additional_special_tokens_ids"""						,		[]			)
																				self.assertListEqual(getattr(_SCREAMING_SNAKE_CASE						,		"""additional_special_tokens"""			)						,		[]			)
																				self.assertListEqual(getattr(_SCREAMING_SNAKE_CASE						,		"""additional_special_tokens_ids"""			)						,		[]			)

																				setattr(_SCREAMING_SNAKE_CASE						,		"""additional_special_tokens_ids"""						,		[token_id_to_test_setters]			)
																				self.assertListEqual(getattr(_SCREAMING_SNAKE_CASE						,		"""additional_special_tokens"""			)						,		[token_to_test_setters]			)
																				self.assertListEqual(getattr(_SCREAMING_SNAKE_CASE						,		"""additional_special_tokens_ids"""			)						,		[token_id_to_test_setters]			)





def        _SCREAMING_SNAKE_CASE      (		snake_case			,     snake_case       )						->				bool:
						_UpperCAmelCase      =			len(snake_case       ) + 1
						_UpperCAmelCase      =			len(snake_case       ) + 1

						# dp is a 2d matrix where dp[i][j] denotes whether prefix string of
						# length i of input_string matches with prefix string of length j of
						# given pattern.
						# "dp" stands for dynamic programming.
						_UpperCAmelCase      =			[[0 for i in range(snake_case       )] for j in range(snake_case       )]

						# since string of zero length match pattern of zero length
						_UpperCAmelCase      =			1

						# since pattern of zero length will never match with string of non-zero length
						for i in range(1			,     snake_case       ):
												_UpperCAmelCase      =			0

						# since string of zero length will match with pattern where there
						# is at least one * alternatively
						for j in range(1			,     snake_case       ):
												_UpperCAmelCase      =			dp[0][j - 2] if pattern[j - 1] == """*""" else 0

						# now using bottom-up approach to find for all remaining lengths
						for i in range(1			,     snake_case       ):
												for j in range(1			,     snake_case       ):
																		if input_string[i - 1] == pattern[j - 1] or pattern[j - 1] == ".":
																								_UpperCAmelCase      =			dp[i - 1][j - 1]

																		elif pattern[j - 1] == "*":
																								if dp[i][j - 2] == 1:
																														_UpperCAmelCase      =			1
																								elif pattern[j - 2] in (input_string[i - 1], "."):
																														_UpperCAmelCase      =			dp[i - 1][j]
																								else:
																														_UpperCAmelCase      =			0
																		else:
																								_UpperCAmelCase      =			0

						return bool(dp[-1][-1]       )


if __name__ == "__main__":
				import doctest

				doctest.testmod()
				# inputing the strings
				# input_string = input("input a string :")
				# pattern = input("input a pattern :")

				a													=			"aab"
				a													=			"c*a*b"

				# using function to check whether given string matches the given pattern
				if match_pattern(input_string, pattern):
								print(F'{input_string} matches the given pattern {pattern}')
				else:
								print(F'{input_string} does not match with the given pattern {pattern}')