





"""simple docstring"""





import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase__     			=			get_tests_dir('''fixtures/test_sentencepiece_no_bos.model''')





@require_sentencepiece
@require_tokenizers
class        _lowerCamelCase      (       UpperCamelCase__  ,			unittest.TestCase ):
						UpperCAmelCase_ =			PegasusTokenizer
						UpperCAmelCase_ =			PegasusTokenizerFast
						UpperCAmelCase_ =			True
						UpperCAmelCase_ =			True


						def     snake_case_   (self		) ->   int:
								super().setUp()

								# We have a SentencePiece fixture for testing
								UpperCamelCase   =	PegasusTokenizer(_a		)
								tokenizer.save_pretrained(self.tmpdirname		)


						@cached_property
						def     snake_case_   (self		) ->   int:
								return PegasusTokenizer.from_pretrained("google/pegasus-large"		)


						def     snake_case_   (self   ,							**__a		) ->   PegasusTokenizer:
								return PegasusTokenizer.from_pretrained(self.tmpdirname   ,							**_a		)


						def     snake_case_   (self   ,							__a		) ->   List[Any]:
								return ("This is a test", "This is a test")


						def     snake_case_   (self		) ->   int:
								UpperCamelCase   =	"""</s>"""
								UpperCamelCase   =	1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a		)   ,							_a		)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a		)   ,							_a		)


						def     snake_case_   (self		) ->   Dict:
								UpperCamelCase   =	list(self.get_tokenizer().get_vocab().keys()		)

								self.assertEqual(vocab_keys[0]   ,							"<pad>"		)
								self.assertEqual(vocab_keys[1]   ,							"</s>"		)
								self.assertEqual(vocab_keys[-1]   ,							"v"		)
								self.assertEqual(len(_a		)   ,							11_03		)


						def     snake_case_   (self		) ->   Optional[int]:
								self.assertEqual(self.get_tokenizer().vocab_size   ,							11_03		)


						def     snake_case_   (self		) ->   Tuple:
								UpperCamelCase   =	self.rust_tokenizer_class.from_pretrained(self.tmpdirname		)
								UpperCamelCase   =	self.tokenizer_class.from_pretrained(self.tmpdirname		)
								UpperCamelCase   =	(
								    """Let's see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important"""
								    """ </s> <pad> <pad> <pad>"""
								)
								UpperCamelCase   =	rust_tokenizer([raw_input_str]   ,							return_tensors=_a   ,							add_special_tokens=_a		).input_ids[0]
								UpperCamelCase   =	py_tokenizer([raw_input_str]   ,							return_tensors=_a   ,							add_special_tokens=_a		).input_ids[0]
								self.assertListEqual(_a   ,							_a		)


						def     snake_case_   (self		) ->   Any:
								UpperCamelCase   =	self._large_tokenizer
								# <mask_1> masks whole sentence while <mask_2> masks single word
								UpperCamelCase   =	"""<mask_1> To ensure a <mask_2> flow of bank resolutions."""
								UpperCamelCase   =	[2, 4_13, 6_15, 1_14, 3, 19_71, 1_13, 16_79, 1_07_10, 1_07, 1]
								UpperCamelCase   =	tokenizer([raw_input_str]   ,							return_tensors=_a		).input_ids[0]
								self.assertListEqual(_a   ,							_a		)


						def     snake_case_   (self		) ->   List[str]:
								UpperCamelCase   =	self._large_tokenizer
								# The tracebacks for the following asserts are **better** without messages or self.assertEqual
								assert tokenizer.vocab_size == 9_61_03
								assert tokenizer.pad_token_id == 0
								assert tokenizer.eos_token_id == 1
								assert tokenizer.offset == 1_03
								assert tokenizer.unk_token_id == tokenizer.offset + 2 == 1_05
								assert tokenizer.unk_token == "<unk>"
								assert tokenizer.model_max_length == 10_24
								UpperCamelCase   =	"""To ensure a smooth flow of bank resolutions."""
								UpperCamelCase   =	[4_13, 6_15, 1_14, 22_91, 19_71, 1_13, 16_79, 1_07_10, 1_07, 1]
								UpperCamelCase   =	tokenizer([raw_input_str]   ,							return_tensors=_a		).input_ids[0]
								self.assertListEqual(_a   ,							_a		)
								assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]		) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]


						@require_torch
						def     snake_case_   (self		) ->   List[str]:
								UpperCamelCase   =	["""This is going to be way too long.""" * 1_50, """short example"""]
								UpperCamelCase   =	["""not super long but more than 5 tokens""", """tiny"""]
								UpperCamelCase   =	self._large_tokenizer(_a   ,							padding=_a   ,							truncation=_a   ,							return_tensors="pt"		)
								UpperCamelCase   =	self._large_tokenizer(
								    text_target=_a   ,							max_length=5   ,							padding=_a   ,							truncation=_a   ,							return_tensors="pt"		)

								assert batch.input_ids.shape == (2, 10_24)
								assert batch.attention_mask.shape == (2, 10_24)
								assert targets["input_ids"].shape == (2, 5)
								assert len(_a		) == 2  # input_ids, attention_mask.




						@slow
						def     snake_case_   (self		) ->   Optional[Any]:
								# fmt: off
								UpperCamelCase   =	{"""input_ids""": [[3_89_79, 1_43, 1_84_85, 6_06, 1_30, 2_66_69, 8_76_86, 1_21, 5_41_89, 11_29, 1_11, 2_66_69, 8_76_86, 1_21, 91_14, 1_47_87, 1_21, 1_32_49, 1_58, 5_92, 9_56, 1_21, 1_46_21, 3_15_76, 1_43, 6_26_13, 1_08, 96_88, 9_30, 4_34_30, 1_15_62, 6_26_13, 3_04, 1_08, 1_14_43, 8_97, 1_08, 93_14, 1_74_15, 6_33_99, 1_08, 1_14_43, 76_14, 1_83_16, 1_18, 42_84, 71_48, 1_24_30, 1_43, 14_00, 2_57_03, 1_58, 1_11, 42_84, 71_48, 1_17_72, 1_43, 2_12_97, 10_64, 1_58, 1_22, 2_04, 35_06, 17_54, 11_33, 1_47_87, 15_81, 1_15, 3_32_24, 44_82, 1_11, 13_55, 1_10, 2_91_73, 3_17, 5_08_33, 1_08, 2_01_47, 9_46_65, 1_11, 7_71_98, 1_07, 1], [1_10, 6_26_13, 1_17, 6_38, 1_12, 11_33, 1_21, 2_00_98, 13_55, 7_90_50, 1_38_72, 1_35, 15_96, 5_35_41, 13_52, 1_41, 1_30_39, 55_42, 1_24, 3_02, 5_18, 1_11, 2_68, 29_56, 1_15, 1_49, 44_27, 1_07, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_39, 12_35, 27_99, 1_82_89, 1_77_80, 2_04, 1_09, 94_74, 12_96, 1_07, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=_a   ,							model_name="google/bigbird-pegasus-large-arxiv"   ,							revision="ba85d0851d708441f91440d509690f1ab6353415"   ,							)





@require_sentencepiece
@require_tokenizers
class        _lowerCamelCase      (       UpperCamelCase__  ,			unittest.TestCase ):
						UpperCAmelCase_ =			PegasusTokenizer
						UpperCAmelCase_ =			PegasusTokenizerFast
						UpperCAmelCase_ =			True
						UpperCAmelCase_ =			True


						def     snake_case_   (self		) ->   Optional[Any]:
								super().setUp()

								# We have a SentencePiece fixture for testing
								UpperCamelCase   =	PegasusTokenizer(_a   ,							offset=0   ,							mask_token_sent=_a   ,							mask_token="[MASK]"		)
								tokenizer.save_pretrained(self.tmpdirname		)


						@cached_property
						def     snake_case_   (self		) ->   Optional[Any]:
								return PegasusTokenizer.from_pretrained("google/bigbird-pegasus-large-arxiv"		)


						def     snake_case_   (self   ,							**__a		) ->   PegasusTokenizer:
								return PegasusTokenizer.from_pretrained(self.tmpdirname   ,							**_a		)


						def     snake_case_   (self   ,							__a		) ->   List[str]:
								return ("This is a test", "This is a test")


						def     snake_case_   (self		) ->   List[Any]:
								UpperCamelCase   =	self.rust_tokenizer_class.from_pretrained(self.tmpdirname		)
								UpperCamelCase   =	self.tokenizer_class.from_pretrained(self.tmpdirname		)
								UpperCamelCase   =	(
								    """Let's see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>"""
								    """ <pad> <pad> <pad>"""
								)
								UpperCamelCase   =	rust_tokenizer([raw_input_str]   ,							return_tensors=_a   ,							add_special_tokens=_a		).input_ids[0]
								UpperCamelCase   =	py_tokenizer([raw_input_str]   ,							return_tensors=_a   ,							add_special_tokens=_a		).input_ids[0]
								self.assertListEqual(_a   ,							_a		)


						@require_torch
						def     snake_case_   (self		) ->   Optional[int]:
								UpperCamelCase   =	["""This is going to be way too long.""" * 10_00, """short example"""]
								UpperCamelCase   =	["""not super long but more than 5 tokens""", """tiny"""]
								UpperCamelCase   =	self._large_tokenizer(_a   ,							padding=_a   ,							truncation=_a   ,							return_tensors="pt"		)
								UpperCamelCase   =	self._large_tokenizer(
								    text_target=_a   ,							max_length=5   ,							padding=_a   ,							truncation=_a   ,							return_tensors="pt"		)

								assert batch.input_ids.shape == (2, 40_96)
								assert batch.attention_mask.shape == (2, 40_96)
								assert targets["input_ids"].shape == (2, 5)
								assert len(_a		) == 2  # input_ids, attention_mask.




						def     snake_case_   (self		) ->   Dict:
								UpperCamelCase   =	(
								    """This is an example string that is used to test the original TF implementation against the HF"""
								    """ implementation"""
								)

								UpperCamelCase   =	self._large_tokenizer(_a		).input_ids

								self.assertListEqual(
								    _a   ,							[1_82, 1_17, 1_42, 5_87, 42_11, 1_20, 1_17, 2_63, 1_12, 8_04, 1_09, 8_56, 2_50_16, 31_37, 4_64, 1_09, 2_69_55, 31_37, 1]   ,							)













"""simple docstring"""





from typing import List

from .keymap import KEYMAP, get_character


def       a__					(					_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		def decorator(_SCREAMING_SNAKE_CASE    ):
				UpperCamelCase   =	getattr(_SCREAMING_SNAKE_CASE  ,	"handle_key"  ,	[]    )
				handle += [key]
				setattr(_SCREAMING_SNAKE_CASE  ,	"handle_key"  ,	_SCREAMING_SNAKE_CASE    )
				return func

		return decorator


def       a__					(					*_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		def decorator(_SCREAMING_SNAKE_CASE    ):
				UpperCamelCase   =	getattr(_SCREAMING_SNAKE_CASE  ,	"handle_key"  ,	[]    )
				handle += keys
				setattr(_SCREAMING_SNAKE_CASE  ,	"handle_key"  ,	_SCREAMING_SNAKE_CASE    )
				return func

		return decorator





class        _lowerCamelCase      (       _lowercase ):


						def __new__(cls   ,							__a   ,							__a   ,							__a		) ->   Any:
								UpperCamelCase   =	super().__new__(cls   ,							__a   ,							__a   ,							__a		)
								if not hasattr(__a   ,							"key_handler"		):
										setattr(__a   ,							"key_handler"   ,							{}		)
								setattr(__a   ,							"handle_input"   ,							KeyHandler.handle_input		)

								for value in attrs.values():
										UpperCamelCase   =	getattr(__a   ,							"handle_key"   ,							[]		)
										for key in handled_keys:
												UpperCamelCase   =	value
								return new_cls




						@staticmethod
						def     snake_case_   (cls		) ->   Optional[Any]:
								UpperCamelCase   =	get_character()
								if char != KEYMAP["undefined"]:
										UpperCamelCase   =	ord(__a		)
								UpperCamelCase   =	cls.key_handler.get(__a		)
								if handler:
										UpperCamelCase   =	char
										return handler(cls		)
								else:
										return None



def       a__					(					cls    ):




		"""simple docstring"""


		return KeyHandler(cls.__name__  ,	cls.__bases__  ,	cls.__dict__.copy()    )






