


import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
 from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase			=							get_tests_dir('''fixtures/test_sentencepiece.model''')

__lowerCAmelCase			=							{'''target_lang''': '''fi''', '''source_lang''': '''en'''}
__lowerCAmelCase			=							'''>>zh<<'''
__lowerCAmelCase			=							'''Helsinki-NLP/'''

if is_torch_available():
 __lowerCAmelCase			=							'''pt'''
elif is_tf_available():
 __lowerCAmelCase			=							'''tf'''
else:
 __lowerCAmelCase			=							'''jax'''





@require_sentencepiece
class   __a      (   __UpperCamelCase	,	unittest.TestCase				):
 __lowercase			:							str       					=			MarianTokenizer
 __lowercase			:							int       					=			False
 __lowercase			:							str       					=			True
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Tuple:




        '''simple docstring'''
        super().setUp()
        lowercase__:       Union[str, Any]        =       ['</s>', '<unk>', '▁This', '▁is', '▁a', '▁t', 'est', '\u0120', '<pad>']
        lowercase__:       Dict        =       dict(zip(lowerCAmelCase__    ,			range(len(lowerCAmelCase__			)			)			)			)
        lowercase__:       Dict        =       Path(self.tmpdirname			)
        save_json(lowerCAmelCase__    ,			save_dir / VOCAB_FILES_NAMES['vocab']			)
        save_json(lowerCAmelCase__    ,			save_dir / VOCAB_FILES_NAMES['tokenizer_config_file']			)
        if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
               copyfile(lowerCAmelCase__    ,			save_dir / VOCAB_FILES_NAMES['source_spm']			)
               copyfile(lowerCAmelCase__    ,			save_dir / VOCAB_FILES_NAMES['target_spm']			)

        lowercase__:       List[Any]        =       MarianTokenizer.from_pretrained(self.tmpdirname			)
        tokenizer.save_pretrained(self.tmpdirname			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			**lowerCAmelCase__			) ->    MarianTokenizer:




        '''simple docstring'''
        return MarianTokenizer.from_pretrained(self.tmpdirname    ,			**lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    Optional[Any]:




        '''simple docstring'''
        return (
            "This is a test",
            "This is a test",
        )
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Optional[Any]:




        '''simple docstring'''
        lowercase__:       List[Any]        =       '</s>'
        lowercase__:       str        =       0

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__			)    ,			lowerCAmelCase__			)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__			)    ,			lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    int:




        '''simple docstring'''
        lowercase__:       int        =       list(self.get_tokenizer().get_vocab().keys()			)

        self.assertEqual(vocab_keys[0]    ,			'</s>'			)
        self.assertEqual(vocab_keys[1]    ,			'<unk>'			)
        self.assertEqual(vocab_keys[-1]    ,			'<pad>'			)
        self.assertEqual(len(lowerCAmelCase__			)    ,			9			)
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Tuple:




        '''simple docstring'''
        self.assertEqual(self.get_tokenizer().vocab_size    ,			9			)
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    str:




        '''simple docstring'''
        lowercase__:       Optional[int]        =       MarianTokenizer.from_pretrained(F'{ORG_NAME}opus-mt-en-de'			)
        lowercase__:       List[str]        =       en_de_tokenizer(['I am a small frog']    ,			return_tensors=lowerCAmelCase__			)
        self.assertIsInstance(lowerCAmelCase__    ,			lowerCAmelCase__			)
        lowercase__:       List[Any]        =       [38, 121, 14, 697, 38_848, 0]
        self.assertListEqual(lowerCAmelCase__    ,			batch.input_ids[0]			)

        lowercase__:       Tuple        =       tempfile.mkdtemp()
        en_de_tokenizer.save_pretrained(lowerCAmelCase__			)
        lowercase__:       Dict        =       [x.name for x in Path(lowerCAmelCase__			).glob('*'			)]
        self.assertIn('source.spm'    ,			lowerCAmelCase__			)
        MarianTokenizer.from_pretrained(lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    str:




        '''simple docstring'''
        lowercase__:       Optional[int]        =       self.get_tokenizer()

        lowercase__:       Tuple        =       tok(
            ['I am a small frog' * 1_000, 'I am a small frog']    ,			padding=lowerCAmelCase__    ,			truncation=lowerCAmelCase__    ,			return_tensors=lowerCAmelCase__			)
        self.assertIsInstance(lowerCAmelCase__    ,			lowerCAmelCase__			)
        self.assertEqual(batch.input_ids.shape    ,			(2, 512)			)
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Optional[int]:




        '''simple docstring'''
        lowercase__:       Tuple        =       self.get_tokenizer()
        lowercase__:       Optional[int]        =       tok(['I am a tiny frog', 'I am a small frog']    ,			padding=lowerCAmelCase__    ,			return_tensors=lowerCAmelCase__			)
        self.assertIsInstance(lowerCAmelCase__    ,			lowerCAmelCase__			)
        self.assertEqual(batch_smaller.input_ids.shape    ,			(2, 10)			)
 @slow
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    List[Any]:




        '''simple docstring'''
        lowercase__:       Optional[Any]        =       {'input_ids': [[43_495, 462, 20, 42_164, 1_369, 52, 464, 132, 1_703, 492, 13, 7_491, 38_999, 6, 8, 464, 132, 1_703, 492, 13, 4_669, 37_867, 13, 7_525, 27, 1_593, 988, 13, 33_972, 7_029, 6, 20, 8_251, 383, 2, 270, 5_866, 3_788, 2, 2_353, 8_251, 12_338, 2, 13_958, 387, 2, 3_629, 6_953, 188, 2_900, 2, 13_958, 8_011, 11_501, 23, 8_460, 4_073, 34_009, 20, 435, 11_439, 27, 8, 8_460, 4_073, 6_004, 20, 9_988, 375, 27, 33, 266, 1_945, 1_076, 1_350, 37_867, 3_288, 5, 577, 1_076, 4_374, 8, 5_082, 5, 26_453, 257, 556, 403, 2, 242, 132, 383, 316, 492, 8, 10_767, 6, 316, 304, 4_239, 3, 0], [148, 15_722, 19, 1_839, 12, 1_350, 13, 22_327, 5_082, 5_418, 47_567, 35_938, 59, 318, 19_552, 108, 2_183, 54, 14_976, 4_835, 32, 547, 1_114, 8, 315, 2_417, 5, 92, 19_088, 3, 0, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100], [36, 6_395, 12_570, 39_147, 11_597, 6, 266, 4, 45_405, 7_296, 3, 0, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=lowerCAmelCase__    ,			model_name='Helsinki-NLP/opus-mt-en-de'    ,			revision='1a8c2263da11e68e50938f97e10cd57820bd504c'    ,			decode_kwargs={'use_source_tokenizer': True}    ,			)
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    str:




        '''simple docstring'''
        lowercase__:       Any        =       MarianTokenizer.from_pretrained('hf-internal-testing/test-marian-two-vocabs'			)

        lowercase__:       int        =       'Tämä on testi'
        lowercase__:       int        =       'This is a test'

        lowercase__:       List[str]        =       [76, 7, 2_047, 2]
        lowercase__:       int        =       [69, 12, 11, 940, 2]

        lowercase__:       Any        =       tokenizer(lowerCAmelCase__			).input_ids
        self.assertListEqual(lowerCAmelCase__    ,			lowerCAmelCase__			)

        lowercase__:       List[Any]        =       tokenizer(text_target=lowerCAmelCase__			).input_ids
        self.assertListEqual(lowerCAmelCase__    ,			lowerCAmelCase__			)

        lowercase__:       Dict        =       tokenizer.decode(lowerCAmelCase__    ,			skip_special_tokens=lowerCAmelCase__			)
        self.assertEqual(lowerCAmelCase__    ,			lowerCAmelCase__			)





from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCAmelCase			=							logging.get_logger(__name__)

__lowerCAmelCase			=							{
    '''google/pegasus-large''': '''https://huggingface.co/google/pegasus-large/resolve/main/config.json''',
    # See all PEGASUS models at https://huggingface.co/models?filter=pegasus
}





class   __a      (   __UpperCamelCase				):
 __lowercase			:							Any       					=			'pegasus'
 __lowercase			:							Union[str, Any]       					=			['past_key_values']
 __lowercase			:							Any       					=			{'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}
 def __init__(       self    ,			lowerCAmelCase__=50_265    ,			lowerCAmelCase__=1_024    ,			lowerCAmelCase__=12    ,			lowerCAmelCase__=4_096    ,			lowerCAmelCase__=16    ,			lowerCAmelCase__=12    ,			lowerCAmelCase__=4_096    ,			lowerCAmelCase__=16    ,			lowerCAmelCase__=0.0    ,			lowerCAmelCase__=0.0    ,			lowerCAmelCase__=True    ,			lowerCAmelCase__=True    ,			lowerCAmelCase__="gelu"    ,			lowerCAmelCase__=1_024    ,			lowerCAmelCase__=0.1    ,			lowerCAmelCase__=0.0    ,			lowerCAmelCase__=0.0    ,			lowerCAmelCase__=0.0_2    ,			lowerCAmelCase__=0    ,			lowerCAmelCase__=False    ,			lowerCAmelCase__=0    ,			lowerCAmelCase__=1    ,			lowerCAmelCase__=1    ,			**lowerCAmelCase__    ,			) ->    Union[str, Any]:




        '''simple docstring'''
        lowercase__:       int        =       vocab_size
        lowercase__:       Optional[int]        =       max_position_embeddings
        lowercase__:       List[str]        =       d_model
        lowercase__:       Optional[Any]        =       encoder_ffn_dim
        lowercase__:       Optional[Any]        =       encoder_layers
        lowercase__:       Union[str, Any]        =       encoder_attention_heads
        lowercase__:       Optional[int]        =       decoder_ffn_dim
        lowercase__:       Tuple        =       decoder_layers
        lowercase__:       Union[str, Any]        =       decoder_attention_heads
        lowercase__:       Dict        =       dropout
        lowercase__:       List[str]        =       attention_dropout
        lowercase__:       List[str]        =       activation_dropout
        lowercase__:       Optional[int]        =       activation_function
        lowercase__:       Dict        =       init_std
        lowercase__:       Optional[Any]        =       encoder_layerdrop
        lowercase__:       List[str]        =       decoder_layerdrop
        lowercase__:       Union[str, Any]        =       use_cache
        lowercase__:       Any        =       encoder_layers
        lowercase__:       List[str]        =       scale_embedding  # scale factor will be sqrt(d_model) if True
        super().__init__(
            pad_token_id=lowerCAmelCase__    ,			eos_token_id=lowerCAmelCase__    ,			is_encoder_decoder=lowerCAmelCase__    ,			decoder_start_token_id=lowerCAmelCase__    ,			forced_eos_token_id=lowerCAmelCase__    ,			**lowerCAmelCase__    ,			)
 @property
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    int:




        '''simple docstring'''
        return self.encoder_attention_heads
 @property
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    int:




        '''simple docstring'''
        return self.d_model


