


import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu







class lowerCAmelCase_						(			nn.Module	):

 '''simple docstring'''

 def __init__(     self ):
       super().__init__()
       snake_case_	=				nn.Linear(3		,				4 )
       snake_case_	=				nn.BatchNormad(4 )
       snake_case_	=				nn.Linear(4		,				5 )





 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       return self.lineara(self.batchnorm(self.lineara(_UpperCAmelCase ) ) )







class lowerCAmelCase_						(			lowerCamelCase__	):

 '''simple docstring'''

 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase		,				*_UpperCAmelCase		,				**_UpperCAmelCase ):
       return (args[0] + 1,) + args[1:], kwargs







class lowerCAmelCase_						(			lowerCamelCase__	):

 '''simple docstring'''

 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase		,				_UpperCAmelCase ):
       return output + 1







class lowerCAmelCase_						(			unittest.TestCase	):

 '''simple docstring'''

 def 						UpperCamelCase__					(     self ):
       snake_case_	=				ModelForTest()
       snake_case_	=				ModelHook()

       add_hook_to_module(_UpperCAmelCase		,				_UpperCAmelCase )
       self.assertEqual(test_model._hf_hook		,				_UpperCAmelCase )
       self.assertTrue(hasattr(_UpperCAmelCase		,				'''_old_forward''' ) )

       # Check adding the hook did not change the name or the signature
       self.assertEqual(test_model.forward.__name__		,				'''forward''' )
       self.assertListEqual(list(inspect.signature(test_model.forward ).parameters )		,				['''x'''] )

       remove_hook_from_module(_UpperCAmelCase )
       self.assertFalse(hasattr(_UpperCAmelCase		,				'''_hf_hook''' ) )
       self.assertFalse(hasattr(_UpperCAmelCase		,				'''_old_forward''' ) )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				ModelForTest()
       snake_case_	=				ModelHook()

       add_hook_to_module(_UpperCAmelCase		,				_UpperCAmelCase )
       add_hook_to_module(_UpperCAmelCase		,				_UpperCAmelCase		,				append=_UpperCAmelCase )

       self.assertEqual(isinstance(test_model._hf_hook		,				_UpperCAmelCase )		,				_UpperCAmelCase )
       self.assertEqual(len(test_model._hf_hook.hooks )		,				2 )
       self.assertTrue(hasattr(_UpperCAmelCase		,				'''_old_forward''' ) )

       # Check adding the hook did not change the name or the signature
       self.assertEqual(test_model.forward.__name__		,				'''forward''' )
       self.assertListEqual(list(inspect.signature(test_model.forward ).parameters )		,				['''x'''] )

       remove_hook_from_module(_UpperCAmelCase )
       self.assertFalse(hasattr(_UpperCAmelCase		,				'''_hf_hook''' ) )
       self.assertFalse(hasattr(_UpperCAmelCase		,				'''_old_forward''' ) )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				ModelForTest()
       snake_case_	=				torch.randn(2		,				3 )
       snake_case_	=				test_model(x + 1 )
       snake_case_	=				test_model(x + 2 )

       snake_case_	=				PreForwardHook()
       add_hook_to_module(_UpperCAmelCase		,				_UpperCAmelCase )
       snake_case_	=				test_model(_UpperCAmelCase )
       self.assertTrue(torch.allclose(_UpperCAmelCase		,				_UpperCAmelCase		,				atol=1E-5 ) )

       # Attaching a hook to a model when it already has one replaces, does not chain
       snake_case_	=				PreForwardHook()
       add_hook_to_module(_UpperCAmelCase		,				_UpperCAmelCase )
       snake_case_	=				test_model(_UpperCAmelCase )
       self.assertTrue(torch.allclose(_UpperCAmelCase		,				_UpperCAmelCase		,				atol=1E-5 ) )

       # You need to use the sequential hook to chain two or more hooks
       snake_case_	=				SequentialHook(PreForwardHook()		,				PreForwardHook() )
       add_hook_to_module(_UpperCAmelCase		,				_UpperCAmelCase )

       snake_case_	=				test_model(_UpperCAmelCase )
       assert torch.allclose(_UpperCAmelCase		,				_UpperCAmelCase		,				atol=1E-5 )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				ModelForTest()
       snake_case_	=				torch.randn(2		,				3 )
       snake_case_	=				test_model(_UpperCAmelCase )

       snake_case_	=				PostForwardHook()
       add_hook_to_module(_UpperCAmelCase		,				_UpperCAmelCase )
       snake_case_	=				test_model(_UpperCAmelCase )
       self.assertTrue(torch.allclose(_UpperCAmelCase		,				output + 1		,				atol=1E-5 ) )

       # Attaching a hook to a model when it already has one replaces, does not chain
       snake_case_	=				PostForwardHook()
       add_hook_to_module(_UpperCAmelCase		,				_UpperCAmelCase )
       snake_case_	=				test_model(_UpperCAmelCase )
       self.assertTrue(torch.allclose(_UpperCAmelCase		,				output + 1		,				atol=1E-5 ) )

       # You need to use the sequential hook to chain two or more hooks
       snake_case_	=				SequentialHook(PostForwardHook()		,				PostForwardHook() )
       add_hook_to_module(_UpperCAmelCase		,				_UpperCAmelCase )

       snake_case_	=				test_model(_UpperCAmelCase )
       assert torch.allclose(_UpperCAmelCase		,				output + 2		,				atol=1E-5 )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				ModelForTest()
       snake_case_	=				torch.randn(2		,				3 )
       snake_case_	=				test_model(_UpperCAmelCase )

       snake_case_	=				PostForwardHook()
       add_hook_to_module(_UpperCAmelCase		,				_UpperCAmelCase )
       snake_case_	=				test_model(_UpperCAmelCase )
       self.assertTrue(torch.allclose(_UpperCAmelCase		,				output + 1 ) )
       self.assertTrue(outputa.requires_grad )

       snake_case_	=				True
       snake_case_	=				test_model(_UpperCAmelCase )
       self.assertFalse(outputa.requires_grad )



 @require_multi_gpu
 def 						UpperCamelCase__					(     self ):
       snake_case_	=				ModelForTest()
       # Everything is on CPU
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )

       # This will move each submodule on different devices
       add_hook_to_module(model.lineara		,				AlignDevicesHook(execution_device=0 ) )
       add_hook_to_module(model.batchnorm		,				AlignDevicesHook(execution_device=0 ) )
       add_hook_to_module(model.lineara		,				AlignDevicesHook(execution_device=1 ) )

       self.assertEqual(model.lineara.weight.device		,				torch.device(0 ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device(0 ) )
       self.assertEqual(model.batchnorm.running_mean.device		,				torch.device(0 ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device(1 ) )

       # We can still make a forward pass. The input does not need to be on any particular device
       snake_case_	=				torch.randn(2		,				3 )
       snake_case_	=				model(_UpperCAmelCase )
       self.assertEqual(output.device		,				torch.device(1 ) )

       # We can add a general hook to put back output on same device as input.
       add_hook_to_module(_UpperCAmelCase		,				AlignDevicesHook(io_same_device=_UpperCAmelCase ) )
       snake_case_	=				torch.randn(2		,				3 ).to(0 )
       snake_case_	=				model(_UpperCAmelCase )
       self.assertEqual(output.device		,				torch.device(0 ) )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				ModelForTest()

       # Everything is on CPU
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )

       # This will move each submodule on different devices
       snake_case_	=				{'''execution_device''': 0 if torch.cuda.is_available() else '''cpu''', '''offload''': True}

       add_hook_to_module(model.lineara		,				AlignDevicesHook(**_UpperCAmelCase ) )
       add_hook_to_module(model.batchnorm		,				AlignDevicesHook(**_UpperCAmelCase ) )
       add_hook_to_module(model.lineara		,				AlignDevicesHook(**_UpperCAmelCase ) )

       # Parameters have been offloaded, so on the meta device
       self.assertEqual(model.lineara.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''meta''' ) )
       # Buffers are not included in the offload by default, so are on the execution device
       snake_case_	=				torch.device(hook_kwargs['''execution_device'''] )
       self.assertEqual(model.batchnorm.running_mean.device		,				_UpperCAmelCase )

       snake_case_	=				torch.randn(2		,				3 )
       snake_case_	=				model(_UpperCAmelCase )
       self.assertEqual(output.device		,				_UpperCAmelCase )

       # Removing hooks loads back the weights in the model.
       remove_hook_from_module(model.lineara )
       remove_hook_from_module(model.batchnorm )
       remove_hook_from_module(model.lineara )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )

       # Now test with buffers included in the offload
       snake_case_	=				{
           '''execution_device''': 0 if torch.cuda.is_available() else '''cpu''',
           '''offload''': True,
           '''offload_buffers''': True,
       }

       add_hook_to_module(model.lineara		,				AlignDevicesHook(**_UpperCAmelCase ) )
       add_hook_to_module(model.batchnorm		,				AlignDevicesHook(**_UpperCAmelCase ) )
       add_hook_to_module(model.lineara		,				AlignDevicesHook(**_UpperCAmelCase ) )

       # Parameters have been offloaded, so on the meta device, buffers included
       self.assertEqual(model.lineara.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.batchnorm.running_mean.device		,				torch.device('''meta''' ) )

       snake_case_	=				torch.randn(2		,				3 )
       snake_case_	=				model(_UpperCAmelCase )
       self.assertEqual(output.device		,				_UpperCAmelCase )

       # Removing hooks loads back the weights in the model.
       remove_hook_from_module(model.lineara )
       remove_hook_from_module(model.batchnorm )
       remove_hook_from_module(model.lineara )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				ModelForTest()

       # Everything is on CPU
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )

       # This will move each submodule on different devices
       snake_case_	=				0 if torch.cuda.is_available() else '''cpu'''
       attach_align_device_hook(_UpperCAmelCase		,				execution_device=_UpperCAmelCase		,				offload=_UpperCAmelCase )

       # Parameters have been offloaded, so on the meta device
       self.assertEqual(model.lineara.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''meta''' ) )
       # Buffers are not included in the offload by default, so are on the execution device
       snake_case_	=				torch.device(_UpperCAmelCase )
       self.assertEqual(model.batchnorm.running_mean.device		,				_UpperCAmelCase )

       snake_case_	=				torch.randn(2		,				3 )
       snake_case_	=				model(_UpperCAmelCase )
       self.assertEqual(output.device		,				_UpperCAmelCase )

       # Removing hooks loads back the weights in the model.
       remove_hook_from_submodules(_UpperCAmelCase )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )

       # Now test with buffers included in the offload
       attach_align_device_hook(_UpperCAmelCase		,				execution_device=_UpperCAmelCase		,				offload=_UpperCAmelCase		,				offload_buffers=_UpperCAmelCase )

       # Parameters have been offloaded, so on the meta device, buffers included
       self.assertEqual(model.lineara.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.batchnorm.running_mean.device		,				torch.device('''meta''' ) )

       snake_case_	=				torch.randn(2		,				3 )
       snake_case_	=				model(_UpperCAmelCase )
       self.assertEqual(output.device		,				_UpperCAmelCase )

       # Removing hooks loads back the weights in the model.
       remove_hook_from_submodules(_UpperCAmelCase )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )





 def 						UpperCamelCase__					(     self ):
       snake_case_	=				ModelForTest()

       # Everything is on CPU
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )

       # This will move each submodule on different devices
       snake_case_	=				0 if torch.cuda.is_available() else '''cpu'''
       attach_align_device_hook(
           _UpperCAmelCase		,				execution_device=_UpperCAmelCase		,				offload=_UpperCAmelCase		,				weights_map=model.state_dict() )

       # Parameters have been offloaded, so on the meta device
       self.assertEqual(model.lineara.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''meta''' ) )
       # Buffers are not included in the offload by default, so are on the execution device
       snake_case_	=				torch.device(_UpperCAmelCase )
       self.assertEqual(model.batchnorm.running_mean.device		,				_UpperCAmelCase )

       snake_case_	=				torch.randn(2		,				3 )
       snake_case_	=				model(_UpperCAmelCase )
       self.assertEqual(output.device		,				_UpperCAmelCase )

       # Removing hooks loads back the weights in the model.
       remove_hook_from_submodules(_UpperCAmelCase )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )

       # Now test with buffers included in the offload
       attach_align_device_hook(
           _UpperCAmelCase		,				execution_device=_UpperCAmelCase		,				offload=_UpperCAmelCase		,				weights_map=model.state_dict()		,				offload_buffers=_UpperCAmelCase		,				)

       # Parameters have been offloaded, so on the meta device, buffers included
       self.assertEqual(model.lineara.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''meta''' ) )
       self.assertEqual(model.batchnorm.running_mean.device		,				torch.device('''meta''' ) )

       snake_case_	=				torch.randn(2		,				3 )
       snake_case_	=				model(_UpperCAmelCase )
       self.assertEqual(output.device		,				_UpperCAmelCase )

       # Removing hooks loads back the weights in the model.
       remove_hook_from_submodules(_UpperCAmelCase )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.batchnorm.weight.device		,				torch.device('''cpu''' ) )
       self.assertEqual(model.lineara.weight.device		,				torch.device('''cpu''' ) )



import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase      =   get_tests_dir("""fixtures/test_sentencepiece_no_bos.model""")







@require_sentencepiece
@require_tokenizers
class lowerCAmelCase_						(			lowerCamelCase__ ,						unittest.TestCase	):

 '''simple docstring'''

 __snake_case												=  PegasusTokenizer
 __snake_case												=  PegasusTokenizerFast
 __snake_case												=  True
 __snake_case												=  True



 def 						UpperCamelCase__					(     self ):
       super().setUp()

       # We have a SentencePiece fixture for testing
       snake_case_	=				PegasusTokenizer(_UpperCAmelCase )
       tokenizer.save_pretrained(self.tmpdirname )



 @cached_property
 def 						UpperCamelCase__					(     self ):
       return PegasusTokenizer.from_pretrained('''google/pegasus-large''' )



 def 						UpperCamelCase__					(     self		,				**_UpperCAmelCase ):
       return PegasusTokenizer.from_pretrained(self.tmpdirname		,				**_UpperCAmelCase )



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       return ("This is a test", "This is a test")



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				'''</s>'''
       snake_case_	=				1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCAmelCase )		,				_UpperCAmelCase )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCAmelCase )		,				_UpperCAmelCase )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				list(self.get_tokenizer().get_vocab().keys() )

       self.assertEqual(vocab_keys[0]		,				'''<pad>''' )
       self.assertEqual(vocab_keys[1]		,				'''</s>''' )
       self.assertEqual(vocab_keys[-1]		,				'''v''' )
       self.assertEqual(len(_UpperCAmelCase )		,				11_03 )



 def 						UpperCamelCase__					(     self ):
       self.assertEqual(self.get_tokenizer().vocab_size		,				11_03 )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				self.rust_tokenizer_class.from_pretrained(self.tmpdirname )
       snake_case_	=				self.tokenizer_class.from_pretrained(self.tmpdirname )
       snake_case_	=				(
           '''Let\'s see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important'''
           ''' </s> <pad> <pad> <pad>'''
       )
       snake_case_	=				rust_tokenizer([raw_input_str]		,				return_tensors=_UpperCAmelCase		,				add_special_tokens=_UpperCAmelCase ).input_ids[0]
       snake_case_	=				py_tokenizer([raw_input_str]		,				return_tensors=_UpperCAmelCase		,				add_special_tokens=_UpperCAmelCase ).input_ids[0]
       self.assertListEqual(_UpperCAmelCase		,				_UpperCAmelCase )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				self._large_tokenizer
       # <mask_1> masks whole sentence while <mask_2> masks single word
       snake_case_	=				'''<mask_1> To ensure a <mask_2> flow of bank resolutions.'''
       snake_case_	=				[2, 4_13, 6_15, 1_14, 3, 19_71, 1_13, 16_79, 1_07_10, 1_07, 1]
       snake_case_	=				tokenizer([raw_input_str]		,				return_tensors=_UpperCAmelCase ).input_ids[0]
       self.assertListEqual(_UpperCAmelCase		,				_UpperCAmelCase )



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				self._large_tokenizer
       # The tracebacks for the following asserts are **better** without messages or self.assertEqual
       assert tokenizer.vocab_size == 9_61_03
       assert tokenizer.pad_token_id == 0
       assert tokenizer.eos_token_id == 1
       assert tokenizer.offset == 1_03
       assert tokenizer.unk_token_id == tokenizer.offset + 2 == 1_05
       assert tokenizer.unk_token == "<unk>"
       assert tokenizer.model_max_length == 10_24
       snake_case_	=				'''To ensure a smooth flow of bank resolutions.'''
       snake_case_	=				[4_13, 6_15, 1_14, 22_91, 19_71, 1_13, 16_79, 1_07_10, 1_07, 1]
       snake_case_	=				tokenizer([raw_input_str]		,				return_tensors=_UpperCAmelCase ).input_ids[0]
       self.assertListEqual(_UpperCAmelCase		,				_UpperCAmelCase )
       assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3] ) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]



 @require_torch
 def 						UpperCamelCase__					(     self ):
       snake_case_	=				['''This is going to be way too long.''' * 1_50, '''short example''']
       snake_case_	=				['''not super long but more than 5 tokens''', '''tiny''']
       snake_case_	=				self._large_tokenizer(_UpperCAmelCase		,				padding=_UpperCAmelCase		,				truncation=_UpperCAmelCase		,				return_tensors='''pt''' )
       snake_case_	=				self._large_tokenizer(
           text_target=_UpperCAmelCase		,				max_length=5		,				padding=_UpperCAmelCase		,				truncation=_UpperCAmelCase		,				return_tensors='''pt''' )

       assert batch.input_ids.shape == (2, 10_24)
       assert batch.attention_mask.shape == (2, 10_24)
       assert targets["input_ids"].shape == (2, 5)
       assert len(_UpperCAmelCase ) == 2  # input_ids, attention_mask.





 @slow
 def 						UpperCamelCase__					(     self ):
       # fmt: off
       snake_case_	=				{'''input_ids''': [[3_89_79, 1_43, 1_84_85, 6_06, 1_30, 2_66_69, 8_76_86, 1_21, 5_41_89, 11_29, 1_11, 2_66_69, 8_76_86, 1_21, 91_14, 1_47_87, 1_21, 1_32_49, 1_58, 5_92, 9_56, 1_21, 1_46_21, 3_15_76, 1_43, 6_26_13, 1_08, 96_88, 9_30, 4_34_30, 1_15_62, 6_26_13, 3_04, 1_08, 1_14_43, 8_97, 1_08, 93_14, 1_74_15, 6_33_99, 1_08, 1_14_43, 76_14, 1_83_16, 1_18, 42_84, 71_48, 1_24_30, 1_43, 14_00, 2_57_03, 1_58, 1_11, 42_84, 71_48, 1_17_72, 1_43, 2_12_97, 10_64, 1_58, 1_22, 2_04, 35_06, 17_54, 11_33, 1_47_87, 15_81, 1_15, 3_32_24, 44_82, 1_11, 13_55, 1_10, 2_91_73, 3_17, 5_08_33, 1_08, 2_01_47, 9_46_65, 1_11, 7_71_98, 1_07, 1], [1_10, 6_26_13, 1_17, 6_38, 1_12, 11_33, 1_21, 2_00_98, 13_55, 7_90_50, 1_38_72, 1_35, 15_96, 5_35_41, 13_52, 1_41, 1_30_39, 55_42, 1_24, 3_02, 5_18, 1_11, 2_68, 29_56, 1_15, 1_49, 44_27, 1_07, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_39, 12_35, 27_99, 1_82_89, 1_77_80, 2_04, 1_09, 94_74, 12_96, 1_07, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=_UpperCAmelCase		,				model_name='''google/bigbird-pegasus-large-arxiv'''		,				revision='''ba85d0851d708441f91440d509690f1ab6353415'''		,				)







@require_sentencepiece
@require_tokenizers
class lowerCAmelCase_						(			lowerCamelCase__ ,						unittest.TestCase	):

 '''simple docstring'''

 __snake_case												=  PegasusTokenizer
 __snake_case												=  PegasusTokenizerFast
 __snake_case												=  True
 __snake_case												=  True



 def 						UpperCamelCase__					(     self ):
       super().setUp()

       # We have a SentencePiece fixture for testing
       snake_case_	=				PegasusTokenizer(_UpperCAmelCase		,				offset=0		,				mask_token_sent=_UpperCAmelCase		,				mask_token='''[MASK]''' )
       tokenizer.save_pretrained(self.tmpdirname )



 @cached_property
 def 						UpperCamelCase__					(     self ):
       return PegasusTokenizer.from_pretrained('''google/bigbird-pegasus-large-arxiv''' )



 def 						UpperCamelCase__					(     self		,				**_UpperCAmelCase ):
       return PegasusTokenizer.from_pretrained(self.tmpdirname		,				**_UpperCAmelCase )



 def 						UpperCamelCase__					(     self		,				_UpperCAmelCase ):
       return ("This is a test", "This is a test")



 def 						UpperCamelCase__					(     self ):
       snake_case_	=				self.rust_tokenizer_class.from_pretrained(self.tmpdirname )
       snake_case_	=				self.tokenizer_class.from_pretrained(self.tmpdirname )
       snake_case_	=				(
           '''Let\'s see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>'''
           ''' <pad> <pad> <pad>'''
       )
       snake_case_	=				rust_tokenizer([raw_input_str]		,				return_tensors=_UpperCAmelCase		,				add_special_tokens=_UpperCAmelCase ).input_ids[0]
       snake_case_	=				py_tokenizer([raw_input_str]		,				return_tensors=_UpperCAmelCase		,				add_special_tokens=_UpperCAmelCase ).input_ids[0]
       self.assertListEqual(_UpperCAmelCase		,				_UpperCAmelCase )



 @require_torch
 def 						UpperCamelCase__					(     self ):
       snake_case_	=				['''This is going to be way too long.''' * 10_00, '''short example''']
       snake_case_	=				['''not super long but more than 5 tokens''', '''tiny''']
       snake_case_	=				self._large_tokenizer(_UpperCAmelCase		,				padding=_UpperCAmelCase		,				truncation=_UpperCAmelCase		,				return_tensors='''pt''' )
       snake_case_	=				self._large_tokenizer(
           text_target=_UpperCAmelCase		,				max_length=5		,				padding=_UpperCAmelCase		,				truncation=_UpperCAmelCase		,				return_tensors='''pt''' )

       assert batch.input_ids.shape == (2, 40_96)
       assert batch.attention_mask.shape == (2, 40_96)
       assert targets["input_ids"].shape == (2, 5)
       assert len(_UpperCAmelCase ) == 2  # input_ids, attention_mask.





 def 						UpperCamelCase__					(     self ):
       snake_case_	=				(
           '''This is an example string that is used to test the original TF implementation against the HF'''
           ''' implementation'''
       )

       snake_case_	=				self._large_tokenizer(_UpperCAmelCase ).input_ids

       self.assertListEqual(
           _UpperCAmelCase		,				[1_82, 1_17, 1_42, 5_87, 42_11, 1_20, 1_17, 2_63, 1_12, 8_04, 1_09, 8_56, 2_50_16, 31_37, 4_64, 1_09, 2_69_55, 31_37, 1]		,				)