





import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowercase_         =						get_tests_dir("""fixtures/test_sentencepiece_no_bos.model""")
@require_sentencepiece
@require_tokenizers
class 							SCREAMING_SNAKE_CASE  (__SCREAMING_SNAKE_CASE     ,      unittest.TestCase ):
   _UpperCamelCase       :						int					   =			PegasusTokenizer
   _UpperCamelCase       :						int					   =			PegasusTokenizerFast
   _UpperCamelCase       :						Optional[Any]					   =			True
   _UpperCamelCase       :						str					   =			True


   def 			SCREAMING_SNAKE_CASE_				(       self     :							List[Any]			)->  List[Any]:





     """simple docstring"""







     super().setUp()

     # We have a SentencePiece fixture for testing
     lowercase__           =    PegasusTokenizer(_SCREAMING_SNAKE_CASE			)
     tokenizer.save_pretrained(self.tmpdirname			)


   @cached_property
   def 			SCREAMING_SNAKE_CASE_				(       self     :							Dict			)->  Any:





     """simple docstring"""







     return PegasusTokenizer.from_pretrained('google/pegasus-large'			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Dict     ,	**a     :							Union[str, Any]			)->  PegasusTokenizer:





     """simple docstring"""







     return PegasusTokenizer.from_pretrained(self.tmpdirname     ,	**_SCREAMING_SNAKE_CASE			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Optional[int]     ,	a     :							Tuple			)->  Any:





     """simple docstring"""







     return ("This is a test", "This is a test")


   def 			SCREAMING_SNAKE_CASE_				(       self     :							int			)->  Dict:





     """simple docstring"""







     lowercase__           =    "</s>"
     lowercase__           =    1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE			)     ,	_SCREAMING_SNAKE_CASE			)
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE			)     ,	_SCREAMING_SNAKE_CASE			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							List[Any]			)->  Optional[int]:





     """simple docstring"""







     lowercase__           =    list(self.get_tokenizer().get_vocab().keys()			)

     self.assertEqual(vocab_keys[0]     ,	'<pad>'			)
     self.assertEqual(vocab_keys[1]     ,	'</s>'			)
     self.assertEqual(vocab_keys[-1]     ,	'v'			)
     self.assertEqual(len(_SCREAMING_SNAKE_CASE			)     ,	1_103			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							List[Any]			)->  str:





     """simple docstring"""







     self.assertEqual(self.get_tokenizer().vocab_size     ,	1_103			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Tuple			)->  Optional[Any]:





     """simple docstring"""







     lowercase__           =    self.rust_tokenizer_class.from_pretrained(self.tmpdirname			)
     lowercase__           =    self.tokenizer_class.from_pretrained(self.tmpdirname			)
     lowercase__           =    (
         "Let's see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important"
         " </s> <pad> <pad> <pad>"
     )
     lowercase__           =    rust_tokenizer([raw_input_str]     ,	return_tensors=_SCREAMING_SNAKE_CASE     ,	add_special_tokens=_SCREAMING_SNAKE_CASE			).input_ids[0]
     lowercase__           =    py_tokenizer([raw_input_str]     ,	return_tensors=_SCREAMING_SNAKE_CASE     ,	add_special_tokens=_SCREAMING_SNAKE_CASE			).input_ids[0]
     self.assertListEqual(_SCREAMING_SNAKE_CASE     ,	_SCREAMING_SNAKE_CASE			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							str			)->  Union[str, Any]:





     """simple docstring"""







     lowercase__           =    self._large_tokenizer
     # <mask_1> masks whole sentence while <mask_2> masks single word
     lowercase__           =    "<mask_1> To ensure a <mask_2> flow of bank resolutions."
     lowercase__           =    [2, 413, 615, 114, 3, 1_971, 113, 1_679, 10_710, 107, 1]
     lowercase__           =    tokenizer([raw_input_str]     ,	return_tensors=_SCREAMING_SNAKE_CASE			).input_ids[0]
     self.assertListEqual(_SCREAMING_SNAKE_CASE     ,	_SCREAMING_SNAKE_CASE			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Optional[int]			)->  int:





     """simple docstring"""







     lowercase__           =    self._large_tokenizer
     # The tracebacks for the following asserts are **better** without messages or self.assertEqual
     assert tokenizer.vocab_size == 96_103
     assert tokenizer.pad_token_id == 0
     assert tokenizer.eos_token_id == 1
     assert tokenizer.offset == 103
     assert tokenizer.unk_token_id == tokenizer.offset + 2 == 105
     assert tokenizer.unk_token == "<unk>"
     assert tokenizer.model_max_length == 1_024
     lowercase__           =    "To ensure a smooth flow of bank resolutions."
     lowercase__           =    [413, 615, 114, 2_291, 1_971, 113, 1_679, 10_710, 107, 1]
     lowercase__           =    tokenizer([raw_input_str]     ,	return_tensors=_SCREAMING_SNAKE_CASE			).input_ids[0]
     self.assertListEqual(_SCREAMING_SNAKE_CASE     ,	_SCREAMING_SNAKE_CASE			)
     assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]			) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]


   @require_torch
   def 			SCREAMING_SNAKE_CASE_				(       self     :							Dict			)->  Union[str, Any]:





     """simple docstring"""







     lowercase__           =    ["This is going to be way too long." * 150, "short example"]
     lowercase__           =    ["not super long but more than 5 tokens", "tiny"]
     lowercase__           =    self._large_tokenizer(_SCREAMING_SNAKE_CASE     ,	padding=_SCREAMING_SNAKE_CASE     ,	truncation=_SCREAMING_SNAKE_CASE     ,	return_tensors='pt'			)
     lowercase__           =    self._large_tokenizer(
         text_target=_SCREAMING_SNAKE_CASE     ,	max_length=5     ,	padding=_SCREAMING_SNAKE_CASE     ,	truncation=_SCREAMING_SNAKE_CASE     ,	return_tensors='pt'			)

     assert batch.input_ids.shape == (2, 1_024)
     assert batch.attention_mask.shape == (2, 1_024)
     assert targets["input_ids"].shape == (2, 5)
     assert len(_SCREAMING_SNAKE_CASE			) == 2  # input_ids, attention_mask.


   @slow
   def 			SCREAMING_SNAKE_CASE_				(       self     :							Tuple			)->  Optional[Any]:





     """simple docstring"""







     lowercase__           =    {"input_ids": [[38_979, 143, 18_485, 606, 130, 26_669, 87_686, 121, 54_189, 1_129, 111, 26_669, 87_686, 121, 9_114, 14_787, 121, 13_249, 158, 592, 956, 121, 14_621, 31_576, 143, 62_613, 108, 9_688, 930, 43_430, 11_562, 62_613, 304, 108, 11_443, 897, 108, 9_314, 17_415, 63_399, 108, 11_443, 7_614, 18_316, 118, 4_284, 7_148, 12_430, 143, 1_400, 25_703, 158, 111, 4_284, 7_148, 11_772, 143, 21_297, 1_064, 158, 122, 204, 3_506, 1_754, 1_133, 14_787, 1_581, 115, 33_224, 4_482, 111, 1_355, 110, 29_173, 317, 50_833, 108, 20_147, 94_665, 111, 77_198, 107, 1], [110, 62_613, 117, 638, 112, 1_133, 121, 20_098, 1_355, 79_050, 13_872, 135, 1_596, 53_541, 1_352, 141, 13_039, 5_542, 124, 302, 518, 111, 268, 2_956, 115, 149, 4_427, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [139, 1_235, 2_799, 18_289, 17_780, 204, 109, 9_474, 1_296, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=_SCREAMING_SNAKE_CASE     ,	model_name='google/bigbird-pegasus-large-arxiv'     ,	revision='ba85d0851d708441f91440d509690f1ab6353415'     ,	)






@require_sentencepiece
@require_tokenizers
class 							SCREAMING_SNAKE_CASE  (__SCREAMING_SNAKE_CASE     ,      unittest.TestCase ):
   _UpperCamelCase       :						Dict					   =			PegasusTokenizer
   _UpperCamelCase       :						List[Any]					   =			PegasusTokenizerFast
   _UpperCamelCase       :						Any					   =			True
   _UpperCamelCase       :						int					   =			True


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Optional[int]			)->  Union[str, Any]:





     """simple docstring"""







     super().setUp()

     # We have a SentencePiece fixture for testing
     lowercase__           =    PegasusTokenizer(_SCREAMING_SNAKE_CASE     ,	offset=0     ,	mask_token_sent=_SCREAMING_SNAKE_CASE     ,	mask_token='[MASK]'			)
     tokenizer.save_pretrained(self.tmpdirname			)


   @cached_property
   def 			SCREAMING_SNAKE_CASE_				(       self     :							Any			)->  Any:





     """simple docstring"""







     return PegasusTokenizer.from_pretrained('google/bigbird-pegasus-large-arxiv'			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Dict     ,	**a     :							Optional[Any]			)->  PegasusTokenizer:





     """simple docstring"""







     return PegasusTokenizer.from_pretrained(self.tmpdirname     ,	**_SCREAMING_SNAKE_CASE			)


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Any     ,	a     :							Any			)->  Union[str, Any]:





     """simple docstring"""







     return ("This is a test", "This is a test")


   def 			SCREAMING_SNAKE_CASE_				(       self     :							List[str]			)->  Any:





     """simple docstring"""







     lowercase__           =    self.rust_tokenizer_class.from_pretrained(self.tmpdirname			)
     lowercase__           =    self.tokenizer_class.from_pretrained(self.tmpdirname			)
     lowercase__           =    (
         "Let's see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>"
         " <pad> <pad> <pad>"
     )
     lowercase__           =    rust_tokenizer([raw_input_str]     ,	return_tensors=_SCREAMING_SNAKE_CASE     ,	add_special_tokens=_SCREAMING_SNAKE_CASE			).input_ids[0]
     lowercase__           =    py_tokenizer([raw_input_str]     ,	return_tensors=_SCREAMING_SNAKE_CASE     ,	add_special_tokens=_SCREAMING_SNAKE_CASE			).input_ids[0]
     self.assertListEqual(_SCREAMING_SNAKE_CASE     ,	_SCREAMING_SNAKE_CASE			)


   @require_torch
   def 			SCREAMING_SNAKE_CASE_				(       self     :							str			)->  int:





     """simple docstring"""







     lowercase__           =    ["This is going to be way too long." * 1_000, "short example"]
     lowercase__           =    ["not super long but more than 5 tokens", "tiny"]
     lowercase__           =    self._large_tokenizer(_SCREAMING_SNAKE_CASE     ,	padding=_SCREAMING_SNAKE_CASE     ,	truncation=_SCREAMING_SNAKE_CASE     ,	return_tensors='pt'			)
     lowercase__           =    self._large_tokenizer(
         text_target=_SCREAMING_SNAKE_CASE     ,	max_length=5     ,	padding=_SCREAMING_SNAKE_CASE     ,	truncation=_SCREAMING_SNAKE_CASE     ,	return_tensors='pt'			)

     assert batch.input_ids.shape == (2, 4_096)
     assert batch.attention_mask.shape == (2, 4_096)
     assert targets["input_ids"].shape == (2, 5)
     assert len(_SCREAMING_SNAKE_CASE			) == 2  # input_ids, attention_mask.


   def 			SCREAMING_SNAKE_CASE_				(       self     :							Union[str, Any]			)->  int:





     """simple docstring"""







     lowercase__           =    (
         "This is an example string that is used to test the original TF implementation against the HF"
         " implementation"
     )

     lowercase__           =    self._large_tokenizer(_SCREAMING_SNAKE_CASE			).input_ids

     self.assertListEqual(
         _SCREAMING_SNAKE_CASE     ,	[182, 117, 142, 587, 4_211, 120, 117, 263, 112, 804, 109, 856, 25_016, 3_137, 464, 109, 26_955, 3_137, 1]     ,	)







"""simple docstring"""






def      _lowercase							(    __snake_case   ,__snake_case      )		->  int:
			return int((input_a, input_a).count(1      ) != 0      )


def      _lowercase							(    )		->  None:
			assert or_gate(0   ,0      ) == 0
			assert or_gate(0   ,1      ) == 1
			assert or_gate(1   ,0      ) == 1
			assert or_gate(1   ,1      ) == 1


if __name__ == "__main__":
	print(or_gate(0, 1))
	print(or_gate(1, 0))
	print(or_gate(0, 0))
	print(or_gate(1, 1))