




import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
					from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


__A								=       get_tests_dir("fixtures/test_sentencepiece.model")

__A								=       {"target_lang": "fi", "source_lang": "en"}
__A								=       ">>zh<<"
__A								=       "Helsinki-NLP/"

if is_torch_available():
					__A								=       "pt"
elif is_tf_available():
					__A								=       "tf"
else:
					__A								=       "jax"







@require_sentencepiece
class        __lowerCAmelCase					(			__magic_name__     ,							unittest.TestCase					):

			"""simple docstring"""







			snake_case_  						=			MarianTokenizer
			snake_case_  						=			False
			snake_case_  						=			True


			def  lowercase_       ( self				)       -> Any:



							'''simple docstring'''
							super().setUp()
							__lowerCamelCase									=       ['</s>', '<unk>', '▁This', '▁is', '▁a', '▁t', 'est', '\u0120', '<pad>']
							__lowerCamelCase									=       dict(zip(lowerCamelCase__     ,		range(len(lowerCamelCase__				)				)				)				)
							__lowerCamelCase									=       Path(self.tmpdirname				)
							save_json(lowerCamelCase__     ,		save_dir / VOCAB_FILES_NAMES['vocab']				)
							save_json(lowerCamelCase__     ,		save_dir / VOCAB_FILES_NAMES['tokenizer_config_file']				)
							if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
											copyfile(lowerCamelCase__     ,		save_dir / VOCAB_FILES_NAMES['source_spm']				)
											copyfile(lowerCamelCase__     ,		save_dir / VOCAB_FILES_NAMES['target_spm']				)

							__lowerCamelCase									=       MarianTokenizer.from_pretrained(self.tmpdirname				)
							tokenizer.save_pretrained(self.tmpdirname				)


			def  lowercase_       ( self     ,		**lowerCamelCase__				)       -> MarianTokenizer:



							'''simple docstring'''
							return MarianTokenizer.from_pretrained(self.tmpdirname     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__				)       -> Dict:



							'''simple docstring'''
							return (
							    "This is a test",
							    "This is a test",
							)


			def  lowercase_       ( self				)       -> Any:



							'''simple docstring'''
							__lowerCamelCase									=       '</s>'
							__lowerCamelCase									=       0

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase__				)     ,		lowerCamelCase__				)


			def  lowercase_       ( self				)       -> str:



							'''simple docstring'''
							__lowerCamelCase									=       list(self.get_tokenizer().get_vocab().keys()				)

							self.assertEqual(vocab_keys[0]     ,		'</s>'				)
							self.assertEqual(vocab_keys[1]     ,		'<unk>'				)
							self.assertEqual(vocab_keys[-1]     ,		'<pad>'				)
							self.assertEqual(len(lowerCamelCase__				)     ,		9				)


			def  lowercase_       ( self				)       -> Optional[Any]:



							'''simple docstring'''
							self.assertEqual(self.get_tokenizer().vocab_size     ,		9				)


			def  lowercase_       ( self				)       -> List[str]:



							'''simple docstring'''
							__lowerCamelCase									=       MarianTokenizer.from_pretrained(f"""{ORG_NAME}opus-mt-en-de"""				)
							__lowerCamelCase									=       en_de_tokenizer(['I am a small frog']     ,		return_tensors=lowerCamelCase__				)
							self.assertIsInstance(lowerCamelCase__     ,		lowerCamelCase__				)
							__lowerCamelCase									=       [38, 121, 14, 697, 38_848, 0]
							self.assertListEqual(lowerCamelCase__     ,		batch.input_ids[0]				)

							__lowerCamelCase									=       tempfile.mkdtemp()
							en_de_tokenizer.save_pretrained(lowerCamelCase__				)
							__lowerCamelCase									=       [x.name for x in Path(lowerCamelCase__				).glob('*'				)]
							self.assertIn('source.spm'     ,		lowerCamelCase__				)
							MarianTokenizer.from_pretrained(lowerCamelCase__				)


			def  lowercase_       ( self				)       -> int:



							'''simple docstring'''
							__lowerCamelCase									=       self.get_tokenizer()

							__lowerCamelCase									=       tok(
							    ['I am a small frog' * 1_000, 'I am a small frog']     ,		padding=lowerCamelCase__     ,		truncation=lowerCamelCase__     ,		return_tensors=lowerCamelCase__				)
							self.assertIsInstance(lowerCamelCase__     ,		lowerCamelCase__				)
							self.assertEqual(batch.input_ids.shape     ,		(2, 512)				)


			def  lowercase_       ( self				)       -> Any:



							'''simple docstring'''
							__lowerCamelCase									=       self.get_tokenizer()
							__lowerCamelCase									=       tok(['I am a tiny frog', 'I am a small frog']     ,		padding=lowerCamelCase__     ,		return_tensors=lowerCamelCase__				)
							self.assertIsInstance(lowerCamelCase__     ,		lowerCamelCase__				)
							self.assertEqual(batch_smaller.input_ids.shape     ,		(2, 10)				)


			@slow
			def  lowercase_       ( self				)       -> Tuple:



							'''simple docstring'''
							# fmt: off
							__lowerCamelCase									=       {'input_ids': [[43_495, 462, 20, 42_164, 1_369, 52, 464, 132, 1_703, 492, 13, 7_491, 38_999, 6, 8, 464, 132, 1_703, 492, 13, 4_669, 37_867, 13, 7_525, 27, 1_593, 988, 13, 33_972, 7_029, 6, 20, 8_251, 383, 2, 270, 5_866, 3_788, 2, 2_353, 8_251, 12_338, 2, 13_958, 387, 2, 3_629, 6_953, 188, 2_900, 2, 13_958, 8_011, 11_501, 23, 8_460, 4_073, 34_009, 20, 435, 11_439, 27, 8, 8_460, 4_073, 6_004, 20, 9_988, 375, 27, 33, 266, 1_945, 1_076, 1_350, 37_867, 3_288, 5, 577, 1_076, 4_374, 8, 5_082, 5, 26_453, 257, 556, 403, 2, 242, 132, 383, 316, 492, 8, 10_767, 6, 316, 304, 4_239, 3, 0], [148, 15_722, 19, 1_839, 12, 1_350, 13, 22_327, 5_082, 5_418, 47_567, 35_938, 59, 318, 19_552, 108, 2_183, 54, 14_976, 4_835, 32, 547, 1_114, 8, 315, 2_417, 5, 92, 19_088, 3, 0, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100], [36, 6_395, 12_570, 39_147, 11_597, 6, 266, 4, 45_405, 7_296, 3, 0, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=lowerCamelCase__     ,		model_name='Helsinki-NLP/opus-mt-en-de'     ,		revision='1a8c2263da11e68e50938f97e10cd57820bd504c'     ,		decode_kwargs={'use_source_tokenizer': True}     ,		)






			def  lowercase_       ( self				)       -> str:



							'''simple docstring'''
							__lowerCamelCase									=       MarianTokenizer.from_pretrained('hf-internal-testing/test-marian-two-vocabs'				)

							__lowerCamelCase									=       'Tämä on testi'
							__lowerCamelCase									=       'This is a test'

							__lowerCamelCase									=       [76, 7, 2_047, 2]
							__lowerCamelCase									=       [69, 12, 11, 940, 2]

							__lowerCamelCase									=       tokenizer(lowerCamelCase__				).input_ids
							self.assertListEqual(lowerCamelCase__     ,		lowerCamelCase__				)

							__lowerCamelCase									=       tokenizer(text_target=lowerCamelCase__				).input_ids
							self.assertListEqual(lowerCamelCase__     ,		lowerCamelCase__				)

							__lowerCamelCase									=       tokenizer.decode(lowerCamelCase__     ,		skip_special_tokens=lowerCamelCase__				)
							self.assertEqual(lowerCamelCase__     ,		lowerCamelCase__				)








from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__A								=       logging.get_logger(__name__)

__A								=       {
    "facebook/xlm-roberta-xl": "https://huggingface.co/facebook/xlm-roberta-xl/resolve/main/config.json",
    "facebook/xlm-roberta-xxl": "https://huggingface.co/facebook/xlm-roberta-xxl/resolve/main/config.json",
    # See all XLM-RoBERTa-XL models at https://huggingface.co/models?filter=xlm-roberta-xl
}







class        __lowerCAmelCase					(			__magic_name__					):

			"""simple docstring"""







			snake_case_  						=			'''xlm-roberta-xl'''


			def __init__( self     ,		lowerCamelCase__=250_880     ,		lowerCamelCase__=2_560     ,		lowerCamelCase__=36     ,		lowerCamelCase__=32     ,		lowerCamelCase__=10_240     ,		lowerCamelCase__="gelu"     ,		lowerCamelCase__=0.1     ,		lowerCamelCase__=0.1     ,		lowerCamelCase__=514     ,		lowerCamelCase__=1     ,		lowerCamelCase__=0.02     ,		lowerCamelCase__=1e-05     ,		lowerCamelCase__=1     ,		lowerCamelCase__=0     ,		lowerCamelCase__=2     ,		lowerCamelCase__="absolute"     ,		lowerCamelCase__=True     ,		lowerCamelCase__=None     ,		**lowerCamelCase__     ,		)       -> str:



							'''simple docstring'''
							super().__init__(pad_token_id=lowerCamelCase__     ,		bos_token_id=lowerCamelCase__     ,		eos_token_id=lowerCamelCase__     ,		**lowerCamelCase__				)
							__lowerCamelCase									=       vocab_size
							__lowerCamelCase									=       hidden_size
							__lowerCamelCase									=       num_hidden_layers
							__lowerCamelCase									=       num_attention_heads
							__lowerCamelCase									=       hidden_act
							__lowerCamelCase									=       intermediate_size
							__lowerCamelCase									=       hidden_dropout_prob
							__lowerCamelCase									=       attention_probs_dropout_prob
							__lowerCamelCase									=       max_position_embeddings
							__lowerCamelCase									=       type_vocab_size
							__lowerCamelCase									=       initializer_range
							__lowerCamelCase									=       layer_norm_eps
							__lowerCamelCase									=       position_embedding_type
							__lowerCamelCase									=       use_cache
							__lowerCamelCase									=       classifier_dropout







class        __lowerCAmelCase					(			__magic_name__					):

			"""simple docstring"""







			@property
			def  lowercase_       ( self				)       -> Mapping[str, Mapping[int, str]]:



							'''simple docstring'''
							if self.task == "multiple-choice":
											__lowerCamelCase									=       {0: 'batch', 1: 'choice', 2: 'sequence'}
							else:
											__lowerCamelCase									=       {0: 'batch', 1: 'sequence'}
							return OrderedDict(
							    [
							        ('input_ids', dynamic_axis),
							        ('attention_mask', dynamic_axis),
							    ]				)


