

import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__		 =     get_tests_dir('''fixtures/test_sentencepiece_no_bos.model''')

@require_sentencepiece
@require_tokenizers
class     __snake_case    (				snake_case__			,	unittest.TestCase  ):
       """simple docstring"""







       __SCREAMING_SNAKE_CASE				    =							PegasusTokenizer
       __SCREAMING_SNAKE_CASE				    =							PegasusTokenizerFast
       __SCREAMING_SNAKE_CASE				    =							True
       __SCREAMING_SNAKE_CASE				    =							True


       def 			a		(		self   )      ->							List[str]:






              """simple docstring"""


              super().setUp()

              # We have a SentencePiece fixture for testing
              __snake_case       	=							PegasusTokenizer(_UpperCamelCase   )
              tokenizer.save_pretrained(self.tmpdirname   )


       @cached_property
       def 			a		(		self   )      ->							str:






              """simple docstring"""


              return PegasusTokenizer.from_pretrained("""google/pegasus-large"""   )


       def 			a		(		self      , **_UpperCamelCase   )      ->							PegasusTokenizer:






              """simple docstring"""


              return PegasusTokenizer.from_pretrained(self.tmpdirname      , **_UpperCamelCase   )


       def 			a		(		self      , _UpperCamelCase   )      ->							List[Any]:






              """simple docstring"""


              return ("This is a test", "This is a test")


       def 			a		(		self   )      ->							Any:






              """simple docstring"""


              __snake_case       	=							"""</s>"""
              __snake_case       	=							1

              self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCamelCase   )      , _UpperCamelCase   )
              self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCamelCase   )      , _UpperCamelCase   )


       def 			a		(		self   )      ->							Any:






              """simple docstring"""


              __snake_case       	=							list(self.get_tokenizer().get_vocab().keys()   )

              self.assertEqual(vocab_keys[0]      , """<pad>"""   )
              self.assertEqual(vocab_keys[1]      , """</s>"""   )
              self.assertEqual(vocab_keys[-1]      , """v"""   )
              self.assertEqual(len(_UpperCamelCase   )      , 11_03   )


       def 			a		(		self   )      ->							List[str]:






              """simple docstring"""


              self.assertEqual(self.get_tokenizer().vocab_size      , 11_03   )


       def 			a		(		self   )      ->							Optional[Any]:






              """simple docstring"""


              __snake_case       	=							self.rust_tokenizer_class.from_pretrained(self.tmpdirname   )
              __snake_case       	=							self.tokenizer_class.from_pretrained(self.tmpdirname   )
              __snake_case       	=							(
                  """Let's see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important"""
                  """ </s> <pad> <pad> <pad>"""
              )
              __snake_case       	=							rust_tokenizer([raw_input_str]      , return_tensors=_UpperCamelCase      , add_special_tokens=_UpperCamelCase   ).input_ids[0]
              __snake_case       	=							py_tokenizer([raw_input_str]      , return_tensors=_UpperCamelCase      , add_special_tokens=_UpperCamelCase   ).input_ids[0]
              self.assertListEqual(_UpperCamelCase      , _UpperCamelCase   )


       def 			a		(		self   )      ->							Tuple:






              """simple docstring"""


              __snake_case       	=							self._large_tokenizer
              # <mask_1> masks whole sentence while <mask_2> masks single word
              __snake_case       	=							"""<mask_1> To ensure a <mask_2> flow of bank resolutions."""
              __snake_case       	=							[2, 4_13, 6_15, 1_14, 3, 19_71, 1_13, 16_79, 1_07_10, 1_07, 1]
              __snake_case       	=							tokenizer([raw_input_str]      , return_tensors=_UpperCamelCase   ).input_ids[0]
              self.assertListEqual(_UpperCamelCase      , _UpperCamelCase   )


       def 			a		(		self   )      ->							Union[str, Any]:






              """simple docstring"""


              __snake_case       	=							self._large_tokenizer
              # The tracebacks for the following asserts are **better** without messages or self.assertEqual
              assert tokenizer.vocab_size == 9_61_03
              assert tokenizer.pad_token_id == 0
              assert tokenizer.eos_token_id == 1
              assert tokenizer.offset == 1_03
              assert tokenizer.unk_token_id == tokenizer.offset + 2 == 1_05
              assert tokenizer.unk_token == "<unk>"
              assert tokenizer.model_max_length == 10_24
              __snake_case       	=							"""To ensure a smooth flow of bank resolutions."""
              __snake_case       	=							[4_13, 6_15, 1_14, 22_91, 19_71, 1_13, 16_79, 1_07_10, 1_07, 1]
              __snake_case       	=							tokenizer([raw_input_str]      , return_tensors=_UpperCamelCase   ).input_ids[0]
              self.assertListEqual(_UpperCamelCase      , _UpperCamelCase   )
              assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]   ) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]


       @require_torch
       def 			a		(		self   )      ->							Dict:






              """simple docstring"""


              __snake_case       	=							["""This is going to be way too long.""" * 1_50, """short example"""]
              __snake_case       	=							["""not super long but more than 5 tokens""", """tiny"""]
              __snake_case       	=							self._large_tokenizer(_UpperCamelCase      , padding=_UpperCamelCase      , truncation=_UpperCamelCase      , return_tensors="""pt"""   )
              __snake_case       	=							self._large_tokenizer(
                  text_target=_UpperCamelCase      , max_length=5      , padding=_UpperCamelCase      , truncation=_UpperCamelCase      , return_tensors="""pt"""   )

              assert batch.input_ids.shape == (2, 10_24)
              assert batch.attention_mask.shape == (2, 10_24)
              assert targets["input_ids"].shape == (2, 5)
              assert len(_UpperCamelCase   ) == 2  # input_ids, attention_mask.






       @slow
       def 			a		(		self   )      ->							List[str]:






              """simple docstring"""


              __snake_case       	=							{"""input_ids""": [[3_89_79, 1_43, 1_84_85, 6_06, 1_30, 2_66_69, 8_76_86, 1_21, 5_41_89, 11_29, 1_11, 2_66_69, 8_76_86, 1_21, 91_14, 1_47_87, 1_21, 1_32_49, 1_58, 5_92, 9_56, 1_21, 1_46_21, 3_15_76, 1_43, 6_26_13, 1_08, 96_88, 9_30, 4_34_30, 1_15_62, 6_26_13, 3_04, 1_08, 1_14_43, 8_97, 1_08, 93_14, 1_74_15, 6_33_99, 1_08, 1_14_43, 76_14, 1_83_16, 1_18, 42_84, 71_48, 1_24_30, 1_43, 14_00, 2_57_03, 1_58, 1_11, 42_84, 71_48, 1_17_72, 1_43, 2_12_97, 10_64, 1_58, 1_22, 2_04, 35_06, 17_54, 11_33, 1_47_87, 15_81, 1_15, 3_32_24, 44_82, 1_11, 13_55, 1_10, 2_91_73, 3_17, 5_08_33, 1_08, 2_01_47, 9_46_65, 1_11, 7_71_98, 1_07, 1], [1_10, 6_26_13, 1_17, 6_38, 1_12, 11_33, 1_21, 2_00_98, 13_55, 7_90_50, 1_38_72, 1_35, 15_96, 5_35_41, 13_52, 1_41, 1_30_39, 55_42, 1_24, 3_02, 5_18, 1_11, 2_68, 29_56, 1_15, 1_49, 44_27, 1_07, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_39, 12_35, 27_99, 1_82_89, 1_77_80, 2_04, 1_09, 94_74, 12_96, 1_07, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
              # fmt: on

              self.tokenizer_integration_test_util(
                  expected_encoding=_UpperCamelCase      , model_name="""google/bigbird-pegasus-large-arxiv"""      , revision="""ba85d0851d708441f91440d509690f1ab6353415"""      , )






@require_sentencepiece
@require_tokenizers
class     __snake_case    (				snake_case__			,	unittest.TestCase  ):
       """simple docstring"""







       __SCREAMING_SNAKE_CASE				    =							PegasusTokenizer
       __SCREAMING_SNAKE_CASE				    =							PegasusTokenizerFast
       __SCREAMING_SNAKE_CASE				    =							True
       __SCREAMING_SNAKE_CASE				    =							True


       def 			a		(		self   )      ->							str:






              """simple docstring"""


              super().setUp()

              # We have a SentencePiece fixture for testing
              __snake_case       	=							PegasusTokenizer(_UpperCamelCase      , offset=0      , mask_token_sent=_UpperCamelCase      , mask_token="""[MASK]"""   )
              tokenizer.save_pretrained(self.tmpdirname   )


       @cached_property
       def 			a		(		self   )      ->							List[Any]:






              """simple docstring"""


              return PegasusTokenizer.from_pretrained("""google/bigbird-pegasus-large-arxiv"""   )


       def 			a		(		self      , **_UpperCamelCase   )      ->							PegasusTokenizer:






              """simple docstring"""


              return PegasusTokenizer.from_pretrained(self.tmpdirname      , **_UpperCamelCase   )


       def 			a		(		self      , _UpperCamelCase   )      ->							List[str]:






              """simple docstring"""


              return ("This is a test", "This is a test")


       def 			a		(		self   )      ->							Union[str, Any]:






              """simple docstring"""


              __snake_case       	=							self.rust_tokenizer_class.from_pretrained(self.tmpdirname   )
              __snake_case       	=							self.tokenizer_class.from_pretrained(self.tmpdirname   )
              __snake_case       	=							(
                  """Let's see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>"""
                  """ <pad> <pad> <pad>"""
              )
              __snake_case       	=							rust_tokenizer([raw_input_str]      , return_tensors=_UpperCamelCase      , add_special_tokens=_UpperCamelCase   ).input_ids[0]
              __snake_case       	=							py_tokenizer([raw_input_str]      , return_tensors=_UpperCamelCase      , add_special_tokens=_UpperCamelCase   ).input_ids[0]
              self.assertListEqual(_UpperCamelCase      , _UpperCamelCase   )


       @require_torch
       def 			a		(		self   )      ->							List[Any]:






              """simple docstring"""


              __snake_case       	=							["""This is going to be way too long.""" * 10_00, """short example"""]
              __snake_case       	=							["""not super long but more than 5 tokens""", """tiny"""]
              __snake_case       	=							self._large_tokenizer(_UpperCamelCase      , padding=_UpperCamelCase      , truncation=_UpperCamelCase      , return_tensors="""pt"""   )
              __snake_case       	=							self._large_tokenizer(
                  text_target=_UpperCamelCase      , max_length=5      , padding=_UpperCamelCase      , truncation=_UpperCamelCase      , return_tensors="""pt"""   )

              assert batch.input_ids.shape == (2, 40_96)
              assert batch.attention_mask.shape == (2, 40_96)
              assert targets["input_ids"].shape == (2, 5)
              assert len(_UpperCamelCase   ) == 2  # input_ids, attention_mask.






       def 			a		(		self   )      ->							Union[str, Any]:






              """simple docstring"""


              __snake_case       	=							(
                  """This is an example string that is used to test the original TF implementation against the HF"""
                  """ implementation"""
              )

              __snake_case       	=							self._large_tokenizer(_UpperCamelCase   ).input_ids

              self.assertListEqual(
                  _UpperCamelCase      , [1_82, 1_17, 1_42, 5_87, 42_11, 1_20, 1_17, 2_63, 1_12, 8_04, 1_09, 8_56, 2_50_16, 31_37, 4_64, 1_09, 2_69_55, 31_37, 1]      , )




import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_video_inputs


if is_torch_available():
    import torch

if is_vision_available():
    from PIL import Image

    from transformers import VivitImageProcessor

class     __snake_case    (				unittest.TestCase  ):
       """simple docstring"""







       def __init__(		self      , _UpperCamelCase      , _UpperCamelCase=7      , _UpperCamelCase=3      , _UpperCamelCase=10      , _UpperCamelCase=18      , _UpperCamelCase=30      , _UpperCamelCase=4_00      , _UpperCamelCase=True      , _UpperCamelCase=None      , _UpperCamelCase=True      , _UpperCamelCase=[0.5, 0.5, 0.5]      , _UpperCamelCase=[0.5, 0.5, 0.5]      , _UpperCamelCase=None      , )      ->							str:






              """simple docstring"""


              __snake_case       	=							size if size is not None else {"""shortest_edge""": 18}
              __snake_case       	=							crop_size if crop_size is not None else {"""height""": 18, """width""": 18}

              __snake_case       	=							parent
              __snake_case       	=							batch_size
              __snake_case       	=							num_channels
              __snake_case       	=							num_frames
              __snake_case       	=							image_size
              __snake_case       	=							min_resolution
              __snake_case       	=							max_resolution
              __snake_case       	=							do_resize
              __snake_case       	=							size
              __snake_case       	=							do_normalize
              __snake_case       	=							image_mean
              __snake_case       	=							image_std
              __snake_case       	=							crop_size






       def 			a		(		self   )      ->							Optional[Any]:






              """simple docstring"""


              return {
                  "image_mean": self.image_mean,
                  "image_std": self.image_std,
                  "do_normalize": self.do_normalize,
                  "do_resize": self.do_resize,
                  "size": self.size,
                  "crop_size": self.crop_size,
              }






@require_torch
@require_vision
class     __snake_case    (				snake_case__			,	unittest.TestCase  ):
       """simple docstring"""







       __SCREAMING_SNAKE_CASE				    =							VivitImageProcessor if is_vision_available() else None


       def 			a		(		self   )      ->							int:






              """simple docstring"""


              __snake_case       	=							VivitImageProcessingTester(self   )


       @property
       def 			a		(		self   )      ->							str:






              """simple docstring"""


              return self.image_processor_tester.prepare_image_processor_dict()


       def 			a		(		self   )      ->							List[Any]:






              """simple docstring"""


              __snake_case       	=							self.image_processing_class(**self.image_processor_dict   )
              self.assertTrue(hasattr(_UpperCamelCase      , """image_mean"""   )   )
              self.assertTrue(hasattr(_UpperCamelCase      , """image_std"""   )   )
              self.assertTrue(hasattr(_UpperCamelCase      , """do_normalize"""   )   )
              self.assertTrue(hasattr(_UpperCamelCase      , """do_resize"""   )   )
              self.assertTrue(hasattr(_UpperCamelCase      , """do_center_crop"""   )   )
              self.assertTrue(hasattr(_UpperCamelCase      , """size"""   )   )


       def 			a		(		self   )      ->							str:






              """simple docstring"""


              __snake_case       	=							self.image_processing_class.from_dict(self.image_processor_dict   )
              self.assertEqual(image_processor.size      , {"""shortest_edge""": 18}   )
              self.assertEqual(image_processor.crop_size      , {"""height""": 18, """width""": 18}   )

              __snake_case       	=							self.image_processing_class.from_dict(self.image_processor_dict      , size=42      , crop_size=84   )
              self.assertEqual(image_processor.size      , {"""shortest_edge""": 42}   )
              self.assertEqual(image_processor.crop_size      , {"""height""": 84, """width""": 84}   )


       def 			a		(		self   )      ->							List[str]:






              """simple docstring"""


              __snake_case       	=							self.image_processing_class(**self.image_processor_dict   )
              # create random PIL videos
              __snake_case       	=							prepare_video_inputs(self.image_processor_tester      , equal_resolution=_UpperCamelCase   )
              for video in video_inputs:
                     self.assertIsInstance(_UpperCamelCase      , _UpperCamelCase   )
                     self.assertIsInstance(video[0]      , Image.Image   )

              # Test not batched input
              __snake_case       	=							image_processing(video_inputs[0]      , return_tensors="""pt"""   ).pixel_values
              self.assertEqual(
                  encoded_videos.shape      , (
                      1,
                      self.image_processor_tester.num_frames,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["""height"""],
                      self.image_processor_tester.crop_size["""width"""],
                  )      , )

              # Test batched
              __snake_case       	=							image_processing(_UpperCamelCase      , return_tensors="""pt"""   ).pixel_values
              self.assertEqual(
                  encoded_videos.shape      , (
                      self.image_processor_tester.batch_size,
                      self.image_processor_tester.num_frames,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["""height"""],
                      self.image_processor_tester.crop_size["""width"""],
                  )      , )


       def 			a		(		self   )      ->							Union[str, Any]:






              """simple docstring"""


              __snake_case       	=							self.image_processing_class(**self.image_processor_dict   )
              # create random numpy tensors
              __snake_case       	=							prepare_video_inputs(self.image_processor_tester      , equal_resolution=_UpperCamelCase      , numpify=_UpperCamelCase   )
              for video in video_inputs:
                     self.assertIsInstance(_UpperCamelCase      , _UpperCamelCase   )
                     self.assertIsInstance(video[0]      , np.ndarray   )

              # Test not batched input
              __snake_case       	=							image_processing(video_inputs[0]      , return_tensors="""pt"""   ).pixel_values
              self.assertEqual(
                  encoded_videos.shape      , (
                      1,
                      self.image_processor_tester.num_frames,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["""height"""],
                      self.image_processor_tester.crop_size["""width"""],
                  )      , )

              # Test batched
              __snake_case       	=							image_processing(_UpperCamelCase      , return_tensors="""pt"""   ).pixel_values
              self.assertEqual(
                  encoded_videos.shape      , (
                      self.image_processor_tester.batch_size,
                      self.image_processor_tester.num_frames,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["""height"""],
                      self.image_processor_tester.crop_size["""width"""],
                  )      , )






       def 			a		(		self   )      ->							List[Any]:






              """simple docstring"""


              __snake_case       	=							self.image_processing_class(**self.image_processor_dict   )
              # create random PyTorch tensors
              __snake_case       	=							prepare_video_inputs(self.image_processor_tester      , equal_resolution=_UpperCamelCase      , torchify=_UpperCamelCase   )
              for video in video_inputs:
                     self.assertIsInstance(_UpperCamelCase      , _UpperCamelCase   )
                     self.assertIsInstance(video[0]      , torch.Tensor   )

              # Test not batched input
              __snake_case       	=							image_processing(video_inputs[0]      , return_tensors="""pt"""   ).pixel_values
              self.assertEqual(
                  encoded_videos.shape      , (
                      1,
                      self.image_processor_tester.num_frames,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["""height"""],
                      self.image_processor_tester.crop_size["""width"""],
                  )      , )

              # Test batched
              __snake_case       	=							image_processing(_UpperCamelCase      , return_tensors="""pt"""   ).pixel_values
              self.assertEqual(
                  encoded_videos.shape      , (
                      self.image_processor_tester.batch_size,
                      self.image_processor_tester.num_frames,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["""height"""],
                      self.image_processor_tester.crop_size["""width"""],
                  )      , )

