






"""simple docstring"""







import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCamelCase	     =    get_tests_dir("""fixtures/test_sentencepiece_no_bos.model""")







@require_sentencepiece
@require_tokenizers
class    lowerCamelCase__   (			snake_case       ,	unittest.TestCase					):
    SCREAMING_SNAKE_CASE						=     PegasusTokenizer
    SCREAMING_SNAKE_CASE						=     PegasusTokenizerFast
    SCREAMING_SNAKE_CASE						=     True
    SCREAMING_SNAKE_CASE						=     True






    def 							_UpperCamelCase				(					self   ):
          super().setUp()

          # We have a SentencePiece fixture for testing
          UpperCAmelCase							     =			PegasusTokenizer(_lowerCAmelCase   )
          tokenizer.save_pretrained(self.tmpdirname   )






    @cached_property
    def 							_UpperCamelCase				(					self   ):
          return PegasusTokenizer.from_pretrained("""google/pegasus-large"""   )






    def 							_UpperCamelCase				(					self      ,**A   ):
          return PegasusTokenizer.from_pretrained(self.tmpdirname      ,**_lowerCAmelCase   )






    def 							_UpperCamelCase				(					self      ,A   ):
          return ("This is a test", "This is a test")






    def 							_UpperCamelCase				(					self   ):
          UpperCAmelCase							     =			"""</s>"""
          UpperCAmelCase							     =			1

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCAmelCase   )      ,_lowerCAmelCase   )
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCAmelCase   )      ,_lowerCAmelCase   )






    def 							_UpperCamelCase				(					self   ):
          UpperCAmelCase							     =			list(self.get_tokenizer().get_vocab().keys()   )

          self.assertEqual(vocab_keys[0]      ,"""<pad>"""   )
          self.assertEqual(vocab_keys[1]      ,"""</s>"""   )
          self.assertEqual(vocab_keys[-1]      ,"""v"""   )
          self.assertEqual(len(_lowerCAmelCase   )      ,1_103   )






    def 							_UpperCamelCase				(					self   ):
          self.assertEqual(self.get_tokenizer().vocab_size      ,1_103   )






    def 							_UpperCamelCase				(					self   ):
          UpperCAmelCase							     =			self.rust_tokenizer_class.from_pretrained(self.tmpdirname   )
          UpperCAmelCase							     =			self.tokenizer_class.from_pretrained(self.tmpdirname   )
          UpperCAmelCase							     =			(
              """Let's see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important"""
              """ </s> <pad> <pad> <pad>"""
          )
          UpperCAmelCase							     =			rust_tokenizer([raw_input_str]      ,return_tensors=_lowerCAmelCase      ,add_special_tokens=_lowerCAmelCase   ).input_ids[0]
          UpperCAmelCase							     =			py_tokenizer([raw_input_str]      ,return_tensors=_lowerCAmelCase      ,add_special_tokens=_lowerCAmelCase   ).input_ids[0]
          self.assertListEqual(_lowerCAmelCase      ,_lowerCAmelCase   )






    def 							_UpperCamelCase				(					self   ):
          UpperCAmelCase							     =			self._large_tokenizer
          # <mask_1> masks whole sentence while <mask_2> masks single word
          UpperCAmelCase							     =			"""<mask_1> To ensure a <mask_2> flow of bank resolutions."""
          UpperCAmelCase							     =			[2, 413, 615, 114, 3, 1_971, 113, 1_679, 10_710, 107, 1]
          UpperCAmelCase							     =			tokenizer([raw_input_str]      ,return_tensors=_lowerCAmelCase   ).input_ids[0]
          self.assertListEqual(_lowerCAmelCase      ,_lowerCAmelCase   )






    def 							_UpperCamelCase				(					self   ):
          UpperCAmelCase							     =			self._large_tokenizer
          # The tracebacks for the following asserts are **better** without messages or self.assertEqual
          assert tokenizer.vocab_size == 96_103
          assert tokenizer.pad_token_id == 0
          assert tokenizer.eos_token_id == 1
          assert tokenizer.offset == 103
          assert tokenizer.unk_token_id == tokenizer.offset + 2 == 105
          assert tokenizer.unk_token == "<unk>"
          assert tokenizer.model_max_length == 1_024
          UpperCAmelCase							     =			"""To ensure a smooth flow of bank resolutions."""
          UpperCAmelCase							     =			[413, 615, 114, 2_291, 1_971, 113, 1_679, 10_710, 107, 1]
          UpperCAmelCase							     =			tokenizer([raw_input_str]      ,return_tensors=_lowerCAmelCase   ).input_ids[0]
          self.assertListEqual(_lowerCAmelCase      ,_lowerCAmelCase   )
          assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]   ) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]






    @require_torch
    def 							_UpperCamelCase				(					self   ):
          UpperCAmelCase							     =			["""This is going to be way too long.""" * 150, """short example"""]
          UpperCAmelCase							     =			["""not super long but more than 5 tokens""", """tiny"""]
          UpperCAmelCase							     =			self._large_tokenizer(_lowerCAmelCase      ,padding=_lowerCAmelCase      ,truncation=_lowerCAmelCase      ,return_tensors="""pt"""   )
          UpperCAmelCase							     =			self._large_tokenizer(
              text_target=_lowerCAmelCase      ,max_length=5      ,padding=_lowerCAmelCase      ,truncation=_lowerCAmelCase      ,return_tensors="""pt"""   )

          assert batch.input_ids.shape == (2, 1_024)
          assert batch.attention_mask.shape == (2, 1_024)
          assert targets["input_ids"].shape == (2, 5)
          assert len(_lowerCAmelCase   ) == 2  # input_ids, attention_mask.






    @slow
    def 							_UpperCamelCase				(					self   ):
          # fmt: off
          UpperCAmelCase							     =			{"""input_ids""": [[38_979, 143, 18_485, 606, 130, 26_669, 87_686, 121, 54_189, 1_129, 111, 26_669, 87_686, 121, 9_114, 14_787, 121, 13_249, 158, 592, 956, 121, 14_621, 31_576, 143, 62_613, 108, 9_688, 930, 43_430, 11_562, 62_613, 304, 108, 11_443, 897, 108, 9_314, 17_415, 63_399, 108, 11_443, 7_614, 18_316, 118, 4_284, 7_148, 12_430, 143, 1_400, 25_703, 158, 111, 4_284, 7_148, 11_772, 143, 21_297, 1_064, 158, 122, 204, 3_506, 1_754, 1_133, 14_787, 1_581, 115, 33_224, 4_482, 111, 1_355, 110, 29_173, 317, 50_833, 108, 20_147, 94_665, 111, 77_198, 107, 1], [110, 62_613, 117, 638, 112, 1_133, 121, 20_098, 1_355, 79_050, 13_872, 135, 1_596, 53_541, 1_352, 141, 13_039, 5_542, 124, 302, 518, 111, 268, 2_956, 115, 149, 4_427, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [139, 1_235, 2_799, 18_289, 17_780, 204, 109, 9_474, 1_296, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=_lowerCAmelCase      ,model_name="""google/bigbird-pegasus-large-arxiv"""      ,revision="""ba85d0851d708441f91440d509690f1ab6353415"""      ,)







@require_sentencepiece
@require_tokenizers
class    lowerCamelCase__   (			snake_case       ,	unittest.TestCase					):
    SCREAMING_SNAKE_CASE						=     PegasusTokenizer
    SCREAMING_SNAKE_CASE						=     PegasusTokenizerFast
    SCREAMING_SNAKE_CASE						=     True
    SCREAMING_SNAKE_CASE						=     True






    def 							_UpperCamelCase				(					self   ):
          super().setUp()

          # We have a SentencePiece fixture for testing
          UpperCAmelCase							     =			PegasusTokenizer(_lowerCAmelCase      ,offset=0      ,mask_token_sent=_lowerCAmelCase      ,mask_token="""[MASK]"""   )
          tokenizer.save_pretrained(self.tmpdirname   )






    @cached_property
    def 							_UpperCamelCase				(					self   ):
          return PegasusTokenizer.from_pretrained("""google/bigbird-pegasus-large-arxiv"""   )






    def 							_UpperCamelCase				(					self      ,**A   ):
          return PegasusTokenizer.from_pretrained(self.tmpdirname      ,**_lowerCAmelCase   )






    def 							_UpperCamelCase				(					self      ,A   ):
          return ("This is a test", "This is a test")






    def 							_UpperCamelCase				(					self   ):
          UpperCAmelCase							     =			self.rust_tokenizer_class.from_pretrained(self.tmpdirname   )
          UpperCAmelCase							     =			self.tokenizer_class.from_pretrained(self.tmpdirname   )
          UpperCAmelCase							     =			(
              """Let's see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>"""
              """ <pad> <pad> <pad>"""
          )
          UpperCAmelCase							     =			rust_tokenizer([raw_input_str]      ,return_tensors=_lowerCAmelCase      ,add_special_tokens=_lowerCAmelCase   ).input_ids[0]
          UpperCAmelCase							     =			py_tokenizer([raw_input_str]      ,return_tensors=_lowerCAmelCase      ,add_special_tokens=_lowerCAmelCase   ).input_ids[0]
          self.assertListEqual(_lowerCAmelCase      ,_lowerCAmelCase   )






    @require_torch
    def 							_UpperCamelCase				(					self   ):
          UpperCAmelCase							     =			["""This is going to be way too long.""" * 1_000, """short example"""]
          UpperCAmelCase							     =			["""not super long but more than 5 tokens""", """tiny"""]
          UpperCAmelCase							     =			self._large_tokenizer(_lowerCAmelCase      ,padding=_lowerCAmelCase      ,truncation=_lowerCAmelCase      ,return_tensors="""pt"""   )
          UpperCAmelCase							     =			self._large_tokenizer(
              text_target=_lowerCAmelCase      ,max_length=5      ,padding=_lowerCAmelCase      ,truncation=_lowerCAmelCase      ,return_tensors="""pt"""   )

          assert batch.input_ids.shape == (2, 4_096)
          assert batch.attention_mask.shape == (2, 4_096)
          assert targets["input_ids"].shape == (2, 5)
          assert len(_lowerCAmelCase   ) == 2  # input_ids, attention_mask.






    def 							_UpperCamelCase				(					self   ):
          UpperCAmelCase							     =			(
              """This is an example string that is used to test the original TF implementation against the HF"""
              """ implementation"""
          )

          UpperCAmelCase							     =			self._large_tokenizer(_lowerCAmelCase   ).input_ids

          self.assertListEqual(
              _lowerCAmelCase      ,[182, 117, 142, 587, 4_211, 120, 117, 263, 112, 804, 109, 856, 25_016, 3_137, 464, 109, 26_955, 3_137, 1]      ,)










'''simple docstring'''


def 						__a(SCREAMING_SNAKE_CASE_	:       int		,							SCREAMING_SNAKE_CASE_	:       int     ):



    '''simple docstring'''





    return numa ^ numa < 0


if __name__ == "__main__":
       import doctest

       doctest.testmod()


