




'''simple docstring'''







import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_     						=				get_tests_dir('''fixtures/test_sentencepiece_no_bos.model''')

@require_sentencepiece
@require_tokenizers
class 	_snake_case(			_a       , unittest.TestCase      ):
 __snake_case:							str						=					PegasusTokenizer
 __snake_case:							Optional[int]						=					PegasusTokenizerFast
 __snake_case:							str						=					True
 __snake_case:							List[str]						=					True






 def 						_UpperCamelCase (self       :  Optional[int]    )       ->   Tuple:






    """simple docstring"""


    super().setUp()

    # We have a SentencePiece fixture for testing
    A__       =     PegasusTokenizer(a    )
    tokenizer.save_pretrained(self.tmpdirname    )






 @cached_property
 def 						_UpperCamelCase (self       :  int    )       ->   str:






    """simple docstring"""


    return PegasusTokenizer.from_pretrained('google/pegasus-large'    )






 def 						_UpperCamelCase (self       :  Tuple    ,		**a       :  str    )       ->   PegasusTokenizer:






    """simple docstring"""


    return PegasusTokenizer.from_pretrained(self.tmpdirname    ,		**a    )






 def 						_UpperCamelCase (self       :  Any    ,		a       :  Optional[int]    )       ->   List[str]:






    """simple docstring"""


    return ("This is a test", "This is a test")






 def 						_UpperCamelCase (self       :  List[str]    )       ->   Union[str, Any]:






    """simple docstring"""


    A__       =     '</s>'
    A__       =     1

    self.assertEqual(self.get_tokenizer()._convert_token_to_id(a    )    ,		a    )
    self.assertEqual(self.get_tokenizer()._convert_id_to_token(a    )    ,		a    )






 def 						_UpperCamelCase (self       :  int    )       ->   str:






    """simple docstring"""


    A__       =     list(self.get_tokenizer().get_vocab().keys()    )

    self.assertEqual(vocab_keys[0]    ,		'<pad>'    )
    self.assertEqual(vocab_keys[1]    ,		'</s>'    )
    self.assertEqual(vocab_keys[-1]    ,		'v'    )
    self.assertEqual(len(a    )    ,		11_03    )






 def 						_UpperCamelCase (self       :  Tuple    )       ->   int:






    """simple docstring"""


    self.assertEqual(self.get_tokenizer().vocab_size    ,		11_03    )






 def 						_UpperCamelCase (self       :  Union[str, Any]    )       ->   List[Any]:






    """simple docstring"""


    A__       =     self.rust_tokenizer_class.from_pretrained(self.tmpdirname    )
    A__       =     self.tokenizer_class.from_pretrained(self.tmpdirname    )
    A__       =     (
        'Let\'s see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important'
        ' </s> <pad> <pad> <pad>'
    )
    A__       =     rust_tokenizer([raw_input_str]    ,		return_tensors=a    ,		add_special_tokens=a    ).input_ids[0]
    A__       =     py_tokenizer([raw_input_str]    ,		return_tensors=a    ,		add_special_tokens=a    ).input_ids[0]
    self.assertListEqual(a    ,		a    )






 def 						_UpperCamelCase (self       :  Tuple    )       ->   int:






    """simple docstring"""


    A__       =     self._large_tokenizer
    # <mask_1> masks whole sentence while <mask_2> masks single word
    A__       =     '<mask_1> To ensure a <mask_2> flow of bank resolutions.'
    A__       =     [2, 4_13, 6_15, 1_14, 3, 19_71, 1_13, 16_79, 1_07_10, 1_07, 1]
    A__       =     tokenizer([raw_input_str]    ,		return_tensors=a    ).input_ids[0]
    self.assertListEqual(a    ,		a    )






 def 						_UpperCamelCase (self       :  Tuple    )       ->   Optional[Any]:






    """simple docstring"""


    A__       =     self._large_tokenizer
    # The tracebacks for the following asserts are **better** without messages or self.assertEqual
    assert tokenizer.vocab_size == 9_61_03
    assert tokenizer.pad_token_id == 0
    assert tokenizer.eos_token_id == 1
    assert tokenizer.offset == 1_03
    assert tokenizer.unk_token_id == tokenizer.offset + 2 == 1_05
    assert tokenizer.unk_token == "<unk>"
    assert tokenizer.model_max_length == 10_24
    A__       =     'To ensure a smooth flow of bank resolutions.'
    A__       =     [4_13, 6_15, 1_14, 22_91, 19_71, 1_13, 16_79, 1_07_10, 1_07, 1]
    A__       =     tokenizer([raw_input_str]    ,		return_tensors=a    ).input_ids[0]
    self.assertListEqual(a    ,		a    )
    assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]    ) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]






 @require_torch
 def 						_UpperCamelCase (self       :  List[str]    )       ->   int:






    """simple docstring"""


    A__       =     ['This is going to be way too long.' * 1_50, 'short example']
    A__       =     ['not super long but more than 5 tokens', 'tiny']
    A__       =     self._large_tokenizer(a    ,		padding=a    ,		truncation=a    ,		return_tensors='pt'    )
    A__       =     self._large_tokenizer(
        text_target=a    ,		max_length=5    ,		padding=a    ,		truncation=a    ,		return_tensors='pt'    )

    assert batch.input_ids.shape == (2, 10_24)
    assert batch.attention_mask.shape == (2, 10_24)
    assert targets["input_ids"].shape == (2, 5)
    assert len(a    ) == 2  # input_ids, attention_mask.






 @slow
 def 						_UpperCamelCase (self       :  List[str]    )       ->   Tuple:






    """simple docstring"""


    A__       =     {'input_ids': [[3_89_79, 1_43, 1_84_85, 6_06, 1_30, 2_66_69, 8_76_86, 1_21, 5_41_89, 11_29, 1_11, 2_66_69, 8_76_86, 1_21, 91_14, 1_47_87, 1_21, 1_32_49, 1_58, 5_92, 9_56, 1_21, 1_46_21, 3_15_76, 1_43, 6_26_13, 1_08, 96_88, 9_30, 4_34_30, 1_15_62, 6_26_13, 3_04, 1_08, 1_14_43, 8_97, 1_08, 93_14, 1_74_15, 6_33_99, 1_08, 1_14_43, 76_14, 1_83_16, 1_18, 42_84, 71_48, 1_24_30, 1_43, 14_00, 2_57_03, 1_58, 1_11, 42_84, 71_48, 1_17_72, 1_43, 2_12_97, 10_64, 1_58, 1_22, 2_04, 35_06, 17_54, 11_33, 1_47_87, 15_81, 1_15, 3_32_24, 44_82, 1_11, 13_55, 1_10, 2_91_73, 3_17, 5_08_33, 1_08, 2_01_47, 9_46_65, 1_11, 7_71_98, 1_07, 1], [1_10, 6_26_13, 1_17, 6_38, 1_12, 11_33, 1_21, 2_00_98, 13_55, 7_90_50, 1_38_72, 1_35, 15_96, 5_35_41, 13_52, 1_41, 1_30_39, 55_42, 1_24, 3_02, 5_18, 1_11, 2_68, 29_56, 1_15, 1_49, 44_27, 1_07, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_39, 12_35, 27_99, 1_82_89, 1_77_80, 2_04, 1_09, 94_74, 12_96, 1_07, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
    # fmt: on

    self.tokenizer_integration_test_util(
        expected_encoding=a    ,		model_name='google/bigbird-pegasus-large-arxiv'    ,		revision='ba85d0851d708441f91440d509690f1ab6353415'    ,		)




@require_sentencepiece
@require_tokenizers
class 	_snake_case(			_a       , unittest.TestCase      ):
 __snake_case:							Any						=					PegasusTokenizer
 __snake_case:							int						=					PegasusTokenizerFast
 __snake_case:							Dict						=					True
 __snake_case:							Tuple						=					True






 def 						_UpperCamelCase (self       :  int    )       ->   List[str]:






    """simple docstring"""


    super().setUp()

    # We have a SentencePiece fixture for testing
    A__       =     PegasusTokenizer(a    ,		offset=0    ,		mask_token_sent=a    ,		mask_token='[MASK]'    )
    tokenizer.save_pretrained(self.tmpdirname    )






 @cached_property
 def 						_UpperCamelCase (self       :  List[str]    )       ->   List[Any]:






    """simple docstring"""


    return PegasusTokenizer.from_pretrained('google/bigbird-pegasus-large-arxiv'    )






 def 						_UpperCamelCase (self       :  List[str]    ,		**a       :  Tuple    )       ->   PegasusTokenizer:






    """simple docstring"""


    return PegasusTokenizer.from_pretrained(self.tmpdirname    ,		**a    )






 def 						_UpperCamelCase (self       :  Any    ,		a       :  List[str]    )       ->   Any:






    """simple docstring"""


    return ("This is a test", "This is a test")






 def 						_UpperCamelCase (self       :  List[Any]    )       ->   Optional[Any]:






    """simple docstring"""


    A__       =     self.rust_tokenizer_class.from_pretrained(self.tmpdirname    )
    A__       =     self.tokenizer_class.from_pretrained(self.tmpdirname    )
    A__       =     (
        'Let\'s see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>'
        ' <pad> <pad> <pad>'
    )
    A__       =     rust_tokenizer([raw_input_str]    ,		return_tensors=a    ,		add_special_tokens=a    ).input_ids[0]
    A__       =     py_tokenizer([raw_input_str]    ,		return_tensors=a    ,		add_special_tokens=a    ).input_ids[0]
    self.assertListEqual(a    ,		a    )






 @require_torch
 def 						_UpperCamelCase (self       :  int    )       ->   Dict:






    """simple docstring"""


    A__       =     ['This is going to be way too long.' * 10_00, 'short example']
    A__       =     ['not super long but more than 5 tokens', 'tiny']
    A__       =     self._large_tokenizer(a    ,		padding=a    ,		truncation=a    ,		return_tensors='pt'    )
    A__       =     self._large_tokenizer(
        text_target=a    ,		max_length=5    ,		padding=a    ,		truncation=a    ,		return_tensors='pt'    )

    assert batch.input_ids.shape == (2, 40_96)
    assert batch.attention_mask.shape == (2, 40_96)
    assert targets["input_ids"].shape == (2, 5)
    assert len(a    ) == 2  # input_ids, attention_mask.






 def 						_UpperCamelCase (self       :  int    )       ->   Optional[Any]:






    """simple docstring"""


    A__       =     (
        'This is an example string that is used to test the original TF implementation against the HF'
        ' implementation'
    )

    A__       =     self._large_tokenizer(a    ).input_ids

    self.assertListEqual(
        a    ,		[1_82, 1_17, 1_42, 5_87, 42_11, 1_20, 1_17, 2_63, 1_12, 8_04, 1_09, 8_56, 2_50_16, 31_37, 4_64, 1_09, 2_69_55, 31_37, 1]    ,		)






A				:       Any      =  '\n# Transformers installation\n! pip install transformers datasets\n# To install from source instead of the last release, comment the command above and uncomment the following one.\n# ! pip install git+https://github.com/huggingface/transformers.git\n'

A				:       Optional[int]      =  [{'type': 'code', 'content': INSTALL_CONTENT}]
A				:       Optional[int]      =  {
    '{processor_class}': 'FakeProcessorClass',
    '{model_class}': 'FakeModelClass',
    '{object_class}': 'FakeObjectClass',
}





