






import os
import tempfile
import unittest

from transformers import FlaubertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    FlaubertForMultipleChoice,
				    FlaubertForQuestionAnswering,
				    FlaubertForQuestionAnsweringSimple,
				    FlaubertForSequenceClassification,
				    FlaubertForTokenClassification,
				    FlaubertModel,
				    FlaubertWithLMHeadModel,
				)
				from transformers.models.flaubert.modeling_flaubert import FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST

class 		_snake_case	(	A__       ):






			def __init__(							self		,       a		,       a=13		,       a=7		,       a=True		,       a=True		,       a=True		,       a=True		,       a=True		,       a=False		,       a=False		,       a=False		,       a=2		,       a=99		,       a=0		,       a=32		,       a=5		,       a=4		,       a=0.1		,       a=0.1		,       a=512		,       a=12		,       a=2		,       a=0.02		,       a=3		,       a=4		,       a="last"		,       a=None		,       a=None		,       )   ->							Optional[Any]:
						SCREAMING_SNAKE_CASE							=			parent
						SCREAMING_SNAKE_CASE							=			batch_size
						SCREAMING_SNAKE_CASE							=			seq_length
						SCREAMING_SNAKE_CASE							=			is_training
						SCREAMING_SNAKE_CASE							=			use_input_lengths
						SCREAMING_SNAKE_CASE							=			use_token_type_ids
						SCREAMING_SNAKE_CASE							=			use_labels
						SCREAMING_SNAKE_CASE							=			gelu_activation
						SCREAMING_SNAKE_CASE							=			sinusoidal_embeddings
						SCREAMING_SNAKE_CASE							=			causal
						SCREAMING_SNAKE_CASE							=			asm
						SCREAMING_SNAKE_CASE							=			n_langs
						SCREAMING_SNAKE_CASE							=			vocab_size
						SCREAMING_SNAKE_CASE							=			n_special
						SCREAMING_SNAKE_CASE							=			hidden_size
						SCREAMING_SNAKE_CASE							=			num_hidden_layers
						SCREAMING_SNAKE_CASE							=			num_attention_heads
						SCREAMING_SNAKE_CASE							=			hidden_dropout_prob
						SCREAMING_SNAKE_CASE							=			attention_probs_dropout_prob
						SCREAMING_SNAKE_CASE							=			max_position_embeddings
						SCREAMING_SNAKE_CASE							=			type_vocab_size
						SCREAMING_SNAKE_CASE							=			type_sequence_label_size
						SCREAMING_SNAKE_CASE							=			initializer_range
						SCREAMING_SNAKE_CASE							=			num_labels
						SCREAMING_SNAKE_CASE							=			num_choices
						SCREAMING_SNAKE_CASE							=			summary_type
						SCREAMING_SNAKE_CASE							=			use_proj
						SCREAMING_SNAKE_CASE							=			scope






			def  SCREAMING_SNAKE_CASE__     (							self)   ->							Union[str, Any]:
						SCREAMING_SNAKE_CASE							=			ids_tensor([self.batch_size, self.seq_length]		,       self.vocab_size)
						SCREAMING_SNAKE_CASE							=			random_attention_mask([self.batch_size, self.seq_length])

						SCREAMING_SNAKE_CASE							=			None
						if self.use_input_lengths:
									SCREAMING_SNAKE_CASE							=			(
									    ids_tensor([self.batch_size]		,       vocab_size=2) + self.seq_length - 2
									)  # small variation of seq_length

						SCREAMING_SNAKE_CASE							=			None
						if self.use_token_type_ids:
									SCREAMING_SNAKE_CASE							=			ids_tensor([self.batch_size, self.seq_length]		,       self.n_langs)

						SCREAMING_SNAKE_CASE							=			None
						SCREAMING_SNAKE_CASE							=			None
						SCREAMING_SNAKE_CASE							=			None
						if self.use_labels:
									SCREAMING_SNAKE_CASE							=			ids_tensor([self.batch_size]		,       self.type_sequence_label_size)
									SCREAMING_SNAKE_CASE							=			ids_tensor([self.batch_size, self.seq_length]		,       self.num_labels)
									SCREAMING_SNAKE_CASE							=			ids_tensor([self.batch_size]		,       2).float()
									SCREAMING_SNAKE_CASE							=			ids_tensor([self.batch_size]		,       self.num_choices)

						SCREAMING_SNAKE_CASE							=			self.get_config()

						return (
						    config,
						    input_ids,
						    token_type_ids,
						    input_lengths,
						    sequence_labels,
						    token_labels,
						    is_impossible_labels,
						    choice_labels,
						    input_mask,
						)






			def  SCREAMING_SNAKE_CASE__     (							self)   ->							Union[str, Any]:
						return FlaubertConfig(
						    vocab_size=self.vocab_size		,       n_special=self.n_special		,       emb_dim=self.hidden_size		,       n_layers=self.num_hidden_layers		,       n_heads=self.num_attention_heads		,       dropout=self.hidden_dropout_prob		,       attention_dropout=self.attention_probs_dropout_prob		,       gelu_activation=self.gelu_activation		,       sinusoidal_embeddings=self.sinusoidal_embeddings		,       asm=self.asm		,       causal=self.causal		,       n_langs=self.n_langs		,       max_position_embeddings=self.max_position_embeddings		,       initializer_range=self.initializer_range		,       summary_type=self.summary_type		,       use_proj=self.use_proj		,       )






			def  SCREAMING_SNAKE_CASE__     (							self		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       )   ->							Dict:
						SCREAMING_SNAKE_CASE							=			FlaubertModel(config=a)
						model.to(a)
						model.eval()
						SCREAMING_SNAKE_CASE							=			model(a		,       lengths=a		,       langs=a)
						SCREAMING_SNAKE_CASE							=			model(a		,       langs=a)
						SCREAMING_SNAKE_CASE							=			model(a)
						self.parent.assertEqual(result.last_hidden_state.shape		,       (self.batch_size, self.seq_length, self.hidden_size))






			def  SCREAMING_SNAKE_CASE__     (							self		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       )   ->							str:
						SCREAMING_SNAKE_CASE							=			FlaubertWithLMHeadModel(a)
						model.to(a)
						model.eval()

						SCREAMING_SNAKE_CASE							=			model(a		,       token_type_ids=a		,       labels=a)
						self.parent.assertEqual(result.loss.shape		,       ())
						self.parent.assertEqual(result.logits.shape		,       (self.batch_size, self.seq_length, self.vocab_size))






			def  SCREAMING_SNAKE_CASE__     (							self		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       )   ->							Any:
						SCREAMING_SNAKE_CASE							=			FlaubertForQuestionAnsweringSimple(a)
						model.to(a)
						model.eval()

						SCREAMING_SNAKE_CASE							=			model(a)

						SCREAMING_SNAKE_CASE							=			model(a		,       start_positions=a		,       end_positions=a)
						self.parent.assertEqual(result.start_logits.shape		,       (self.batch_size, self.seq_length))
						self.parent.assertEqual(result.end_logits.shape		,       (self.batch_size, self.seq_length))






			def  SCREAMING_SNAKE_CASE__     (							self		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       )   ->							Dict:
						SCREAMING_SNAKE_CASE							=			FlaubertForQuestionAnswering(a)
						model.to(a)
						model.eval()

						SCREAMING_SNAKE_CASE							=			model(a)

						SCREAMING_SNAKE_CASE							=			model(
						    a		,       start_positions=a		,       end_positions=a		,       cls_index=a		,       is_impossible=a		,       p_mask=a		,       )

						SCREAMING_SNAKE_CASE							=			model(
						    a		,       start_positions=a		,       end_positions=a		,       cls_index=a		,       is_impossible=a		,       )

						((SCREAMING_SNAKE_CASE)					,    )							=			result_with_labels.to_tuple()

						SCREAMING_SNAKE_CASE							=			model(a		,       start_positions=a		,       end_positions=a)

						((SCREAMING_SNAKE_CASE)					,    )							=			result_with_labels.to_tuple()

						self.parent.assertEqual(result_with_labels.loss.shape		,       ())
						self.parent.assertEqual(result.start_top_log_probs.shape		,       (self.batch_size, model.config.start_n_top))
						self.parent.assertEqual(result.start_top_index.shape		,       (self.batch_size, model.config.start_n_top))
						self.parent.assertEqual(
						    result.end_top_log_probs.shape		,       (self.batch_size, model.config.start_n_top * model.config.end_n_top))
						self.parent.assertEqual(
						    result.end_top_index.shape		,       (self.batch_size, model.config.start_n_top * model.config.end_n_top))
						self.parent.assertEqual(result.cls_logits.shape		,       (self.batch_size,))






			def  SCREAMING_SNAKE_CASE__     (							self		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       )   ->							int:
						SCREAMING_SNAKE_CASE							=			FlaubertForSequenceClassification(a)
						model.to(a)
						model.eval()

						SCREAMING_SNAKE_CASE							=			model(a)
						SCREAMING_SNAKE_CASE							=			model(a		,       labels=a)

						self.parent.assertEqual(result.loss.shape		,       ())
						self.parent.assertEqual(result.logits.shape		,       (self.batch_size, self.type_sequence_label_size))






			def  SCREAMING_SNAKE_CASE__     (							self		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       )   ->							Any:
						SCREAMING_SNAKE_CASE							=			self.num_labels
						SCREAMING_SNAKE_CASE							=			FlaubertForTokenClassification(a)
						model.to(a)
						model.eval()

						SCREAMING_SNAKE_CASE							=			model(a		,       attention_mask=a		,       labels=a)
						self.parent.assertEqual(result.logits.shape		,       (self.batch_size, self.seq_length, self.num_labels))






			def  SCREAMING_SNAKE_CASE__     (							self		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       a		,       )   ->							Tuple:
						SCREAMING_SNAKE_CASE							=			self.num_choices
						SCREAMING_SNAKE_CASE							=			FlaubertForMultipleChoice(config=a)
						model.to(a)
						model.eval()
						SCREAMING_SNAKE_CASE							=			input_ids.unsqueeze(1).expand(-1		,       self.num_choices		,       -1).contiguous()
						SCREAMING_SNAKE_CASE							=			token_type_ids.unsqueeze(1).expand(-1		,       self.num_choices		,       -1).contiguous()
						SCREAMING_SNAKE_CASE							=			input_mask.unsqueeze(1).expand(-1		,       self.num_choices		,       -1).contiguous()
						SCREAMING_SNAKE_CASE							=			model(
						    a		,       attention_mask=a		,       token_type_ids=a		,       labels=a		,       )
						self.parent.assertEqual(result.logits.shape		,       (self.batch_size, self.num_choices))







			def  SCREAMING_SNAKE_CASE__     (							self)   ->							Tuple:
						SCREAMING_SNAKE_CASE							=			self.prepare_config_and_inputs()
						(
						    (
						    SCREAMING_SNAKE_CASE
						)					,    (
						    SCREAMING_SNAKE_CASE
						)					,    (
						    SCREAMING_SNAKE_CASE
						)					,    (
						    SCREAMING_SNAKE_CASE
						)					,    (
						    SCREAMING_SNAKE_CASE
						)					,    (
						    SCREAMING_SNAKE_CASE
						)					,    (
						    SCREAMING_SNAKE_CASE
						)					,    (
						    SCREAMING_SNAKE_CASE
						)					,    (
						    SCREAMING_SNAKE_CASE
						)					,    
						)							=			config_and_inputs
						SCREAMING_SNAKE_CASE							=			{
						    'input_ids': input_ids,
						    'token_type_ids': token_type_ids,
						    'lengths': input_lengths,
						    'attention_mask': input_mask,
						}
						return config, inputs_dict



@require_torch
class 		_snake_case	(	A__ ,     A__ ,     unittest.TestCase       ):
			_lowercase		:     Any        =				(
			    (
			        FlaubertModel,
			        FlaubertWithLMHeadModel,
			        FlaubertForQuestionAnswering,
			        FlaubertForQuestionAnsweringSimple,
			        FlaubertForSequenceClassification,
			        FlaubertForTokenClassification,
			        FlaubertForMultipleChoice,
			    )
			    if is_torch_available()
			    else ()
			)
			_lowercase		:     Any        =				(
			    {
			        '''feature-extraction''': FlaubertModel,
			        '''fill-mask''': FlaubertWithLMHeadModel,
			        '''question-answering''': FlaubertForQuestionAnsweringSimple,
			        '''text-classification''': FlaubertForSequenceClassification,
			        '''token-classification''': FlaubertForTokenClassification,
			        '''zero-shot''': FlaubertForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)






			def  SCREAMING_SNAKE_CASE__     (							self		,       a		,       a		,       a		,       a		,       a)   ->							Optional[int]:
						if (
						    pipeline_test_casse_name == "QAPipelineTests"
						    and tokenizer_name is not None
						    and not tokenizer_name.endswith('Fast')
						):
									# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
									# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
									# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
									return True

						return False






			def  SCREAMING_SNAKE_CASE__     (							self		,       a		,       a		,       a=False)   ->							Any:
						SCREAMING_SNAKE_CASE							=			super()._prepare_for_class(a		,       a		,       return_labels=a)

						if return_labels:
									if model_class.__name__ == "FlaubertForQuestionAnswering":
												SCREAMING_SNAKE_CASE							=			torch.zeros(
												    self.model_tester.batch_size		,       dtype=torch.long		,       device=a)
												SCREAMING_SNAKE_CASE							=			torch.zeros(
												    self.model_tester.batch_size		,       dtype=torch.long		,       device=a)

						return inputs_dict






			def  SCREAMING_SNAKE_CASE__     (							self)   ->							Dict:
						SCREAMING_SNAKE_CASE							=			FlaubertModelTester(self)
						SCREAMING_SNAKE_CASE							=			ConfigTester(self		,       config_class=a		,       emb_dim=37)






			def  SCREAMING_SNAKE_CASE__     (							self)   ->							List[Any]:
						self.config_tester.run_common_tests()






			def  SCREAMING_SNAKE_CASE__     (							self)   ->							Any:
						SCREAMING_SNAKE_CASE							=			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_flaubert_model(*a)






			def  SCREAMING_SNAKE_CASE__     (							self)   ->							Optional[Any]:
						SCREAMING_SNAKE_CASE							=			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_flaubert_lm_head(*a)






			def  SCREAMING_SNAKE_CASE__     (							self)   ->							List[str]:
						SCREAMING_SNAKE_CASE							=			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_flaubert_simple_qa(*a)






			def  SCREAMING_SNAKE_CASE__     (							self)   ->							List[str]:
						SCREAMING_SNAKE_CASE							=			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_flaubert_qa(*a)






			def  SCREAMING_SNAKE_CASE__     (							self)   ->							Any:
						SCREAMING_SNAKE_CASE							=			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_flaubert_sequence_classif(*a)






			def  SCREAMING_SNAKE_CASE__     (							self)   ->							Tuple:
						SCREAMING_SNAKE_CASE							=			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_flaubert_token_classif(*a)






			def  SCREAMING_SNAKE_CASE__     (							self)   ->							List[str]:
						SCREAMING_SNAKE_CASE							=			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_flaubert_multiple_choice(*a)






			@slow
			def  SCREAMING_SNAKE_CASE__     (							self)   ->							Tuple:
						for model_name in FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									SCREAMING_SNAKE_CASE							=			FlaubertModel.from_pretrained(a)
									self.assertIsNotNone(a)







			@slow
			@require_torch_gpu
			def  SCREAMING_SNAKE_CASE__     (							self)   ->							Tuple:
						SCREAMING_SNAKE_CASE					,    SCREAMING_SNAKE_CASE							=			self.model_tester.prepare_config_and_inputs_for_common()
						for model_class in self.all_model_classes:
									# FlauBertForMultipleChoice behaves incorrectly in JIT environments.
									if model_class == FlaubertForMultipleChoice:
												return

									SCREAMING_SNAKE_CASE							=			True
									SCREAMING_SNAKE_CASE							=			model_class(config=a)

									SCREAMING_SNAKE_CASE							=			self._prepare_for_class(a		,       a)
									SCREAMING_SNAKE_CASE							=			torch.jit.trace(
									    a		,       (inputs_dict['input_ids'].to('cpu'), inputs_dict['attention_mask'].to('cpu')))

									with tempfile.TemporaryDirectory() as tmp:
												torch.jit.save(a		,       os.path.join(a		,       'traced_model.pt'))
												SCREAMING_SNAKE_CASE							=			torch.jit.load(os.path.join(a		,       'traced_model.pt')		,       map_location=a)
												loaded(inputs_dict['input_ids'].to(a)		,       inputs_dict['attention_mask'].to(a))



@require_torch
class 		_snake_case	(	unittest.TestCase       ):






			@slow
			def  SCREAMING_SNAKE_CASE__     (							self)   ->							List[Any]:
						SCREAMING_SNAKE_CASE							=			FlaubertModel.from_pretrained('flaubert/flaubert_base_cased')
						SCREAMING_SNAKE_CASE							=			torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]])
						with torch.no_grad():
									SCREAMING_SNAKE_CASE							=			model(a)[0]
						SCREAMING_SNAKE_CASE							=			torch.Size((1, 11, 768))
						self.assertEqual(output.shape		,       a)
						SCREAMING_SNAKE_CASE							=			torch.tensor(
						    [[[-2.62_51, -1.42_98, -0.02_27], [-2.85_10, -1.63_87, 0.22_58], [-2.81_14, -1.18_32, -0.30_66]]])

						self.assertTrue(torch.allclose(output[:, :3, :3]		,       a		,       atol=1E-4))









"""simple docstring"""






import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
	from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case				:  List[Any] =       get_tests_dir('fixtures/test_sentencepiece.model')

__snake_case				:  Union[str, Any] =       {'target_lang': 'fi', 'source_lang': 'en'}
__snake_case				:  Union[str, Any] =       '>>zh<<'
__snake_case				:  List[str] =       'Helsinki-NLP/'

if is_torch_available():
	__snake_case				:  Optional[int] =       'pt'
elif is_tf_available():
	__snake_case				:  List[Any] =       'tf'
else:
	__snake_case				:  Union[str, Any] =       'jax'



@require_sentencepiece
class      A__  (    __SCREAMING_SNAKE_CASE    ,						unittest.TestCase				):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE        =      MarianTokenizer
	SCREAMING_SNAKE_CASE        =      False
	SCREAMING_SNAKE_CASE        =      True

	def 				_SCREAMING_SNAKE_CASE  (	self:						int)							->			Tuple:

				"""simple docstring"""

				super().setUp()
				__lowerCAmelCase :    Dict				=			["</s>", "<unk>", "▁This", "▁is", "▁a", "▁t", "est", "\u0120", "<pad>"]
				__lowerCAmelCase :    Union[str, Any]				=			dict(zip(_SCREAMING_SNAKE_CASE			,     range(len(_SCREAMING_SNAKE_CASE))))
				__lowerCAmelCase :    int				=			Path(self.tmpdirname)
				save_json(_SCREAMING_SNAKE_CASE			,     save_dir / VOCAB_FILES_NAMES["vocab"])
				save_json(_SCREAMING_SNAKE_CASE			,     save_dir / VOCAB_FILES_NAMES["tokenizer_config_file"])
				if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
							copyfile(_SCREAMING_SNAKE_CASE			,     save_dir / VOCAB_FILES_NAMES["source_spm"])
							copyfile(_SCREAMING_SNAKE_CASE			,     save_dir / VOCAB_FILES_NAMES["target_spm"])

				__lowerCAmelCase :    Tuple				=			MarianTokenizer.from_pretrained(self.tmpdirname)
				tokenizer.save_pretrained(self.tmpdirname)

	def 				_SCREAMING_SNAKE_CASE  (	self:						Dict			,     **_SCREAMING_SNAKE_CASE:						List[str])							->			MarianTokenizer:

				"""simple docstring"""

				return MarianTokenizer.from_pretrained(self.tmpdirname			,     **_SCREAMING_SNAKE_CASE)

	def 				_SCREAMING_SNAKE_CASE  (	self:						List[Any]			,     _SCREAMING_SNAKE_CASE:						List[str])							->			Dict:

				"""simple docstring"""

				return (
				    "This is a test",
				    "This is a test",
				)

	def 				_SCREAMING_SNAKE_CASE  (	self:						Optional[int])							->			int:

				"""simple docstring"""

				__lowerCAmelCase :    Union[str, Any]				=			"</s>"
				__lowerCAmelCase :    int				=			0

				self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE)			,     _SCREAMING_SNAKE_CASE)
				self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE)			,     _SCREAMING_SNAKE_CASE)

	def 				_SCREAMING_SNAKE_CASE  (	self:						Tuple)							->			List[Any]:

				"""simple docstring"""

				__lowerCAmelCase :    List[str]				=			list(self.get_tokenizer().get_vocab().keys())

				self.assertEqual(vocab_keys[0]			,     "</s>")
				self.assertEqual(vocab_keys[1]			,     "<unk>")
				self.assertEqual(vocab_keys[-1]			,     "<pad>")
				self.assertEqual(len(_SCREAMING_SNAKE_CASE)			,     9)

	def 				_SCREAMING_SNAKE_CASE  (	self:						Tuple)							->			Dict:

				"""simple docstring"""

				self.assertEqual(self.get_tokenizer().vocab_size			,     9)

	def 				_SCREAMING_SNAKE_CASE  (	self:						List[Any])							->			Any:

				"""simple docstring"""

				__lowerCAmelCase :    Optional[Any]				=			MarianTokenizer.from_pretrained(F"""{ORG_NAME}opus-mt-en-de""")
				__lowerCAmelCase :    int				=			en_de_tokenizer(["I am a small frog"]			,     return_tensors=_SCREAMING_SNAKE_CASE)
				self.assertIsInstance(_SCREAMING_SNAKE_CASE			,     _SCREAMING_SNAKE_CASE)
				__lowerCAmelCase :    Optional[int]				=			[38, 121, 14, 697, 3_8848, 0]
				self.assertListEqual(_SCREAMING_SNAKE_CASE			,     batch.input_ids[0])

				__lowerCAmelCase :    Tuple				=			tempfile.mkdtemp()
				en_de_tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase :    List[Any]				=			[x.name for x in Path(_SCREAMING_SNAKE_CASE).glob("*")]
				self.assertIn("source.spm"			,     _SCREAMING_SNAKE_CASE)
				MarianTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE)

	def 				_SCREAMING_SNAKE_CASE  (	self:						Union[str, Any])							->			List[Any]:

				"""simple docstring"""

				__lowerCAmelCase :    List[str]				=			self.get_tokenizer()

				__lowerCAmelCase :    int				=			tok(
				    ["I am a small frog" * 1000, "I am a small frog"]			,     padding=_SCREAMING_SNAKE_CASE			,     truncation=_SCREAMING_SNAKE_CASE			,     return_tensors=_SCREAMING_SNAKE_CASE)
				self.assertIsInstance(_SCREAMING_SNAKE_CASE			,     _SCREAMING_SNAKE_CASE)
				self.assertEqual(batch.input_ids.shape			,     (2, 512))

	def 				_SCREAMING_SNAKE_CASE  (	self:						List[Any])							->			Optional[Any]:

				"""simple docstring"""

				__lowerCAmelCase :    Any				=			self.get_tokenizer()
				__lowerCAmelCase :    str				=			tok(["I am a tiny frog", "I am a small frog"]			,     padding=_SCREAMING_SNAKE_CASE			,     return_tensors=_SCREAMING_SNAKE_CASE)
				self.assertIsInstance(_SCREAMING_SNAKE_CASE			,     _SCREAMING_SNAKE_CASE)
				self.assertEqual(batch_smaller.input_ids.shape			,     (2, 10))

	@slow
	def 				_SCREAMING_SNAKE_CASE  (	self:						Optional[Any])							->			Optional[Any]:

				"""simple docstring"""

				__lowerCAmelCase :    Optional[int]				=			{"input_ids": [[4_3495, 462, 20, 4_2164, 1369, 52, 464, 132, 1703, 492, 13, 7491, 3_8999, 6, 8, 464, 132, 1703, 492, 13, 4669, 3_7867, 13, 7525, 27, 1593, 988, 13, 3_3972, 7029, 6, 20, 8251, 383, 2, 270, 5866, 3788, 2, 2353, 8251, 1_2338, 2, 1_3958, 387, 2, 3629, 6953, 188, 2900, 2, 1_3958, 8011, 1_1501, 23, 8460, 4073, 3_4009, 20, 435, 1_1439, 27, 8, 8460, 4073, 6004, 20, 9988, 375, 27, 33, 266, 1945, 1076, 1350, 3_7867, 3288, 5, 577, 1076, 4374, 8, 5082, 5, 2_6453, 257, 556, 403, 2, 242, 132, 383, 316, 492, 8, 1_0767, 6, 316, 304, 4239, 3, 0], [148, 1_5722, 19, 1839, 12, 1350, 13, 2_2327, 5082, 5418, 4_7567, 3_5938, 59, 318, 1_9552, 108, 2183, 54, 1_4976, 4835, 32, 547, 1114, 8, 315, 2417, 5, 92, 1_9088, 3, 0, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100], [36, 6395, 1_2570, 3_9147, 1_1597, 6, 266, 4, 4_5405, 7296, 3, 0, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
				# fmt: on

				self.tokenizer_integration_test_util(
				    expected_encoding=_SCREAMING_SNAKE_CASE			,     model_name="Helsinki-NLP/opus-mt-en-de"			,     revision="1a8c2263da11e68e50938f97e10cd57820bd504c"			,     decode_kwargs={"use_source_tokenizer": True}			,     )







	def 				_SCREAMING_SNAKE_CASE  (	self:						Union[str, Any])							->			Any:

				"""simple docstring"""

				__lowerCAmelCase :    List[Any]				=			MarianTokenizer.from_pretrained("hf-internal-testing/test-marian-two-vocabs")

				__lowerCAmelCase :    List[str]				=			"Tämä on testi"
				__lowerCAmelCase :    int				=			"This is a test"

				__lowerCAmelCase :    Union[str, Any]				=			[76, 7, 2047, 2]
				__lowerCAmelCase :    Dict				=			[69, 12, 11, 940, 2]

				__lowerCAmelCase :    List[str]				=			tokenizer(_SCREAMING_SNAKE_CASE).input_ids
				self.assertListEqual(_SCREAMING_SNAKE_CASE			,     _SCREAMING_SNAKE_CASE)

				__lowerCAmelCase :    str				=			tokenizer(text_target=_SCREAMING_SNAKE_CASE).input_ids
				self.assertListEqual(_SCREAMING_SNAKE_CASE			,     _SCREAMING_SNAKE_CASE)

				__lowerCAmelCase :    Dict				=			tokenizer.decode(_SCREAMING_SNAKE_CASE			,     skip_special_tokens=_SCREAMING_SNAKE_CASE)
				self.assertEqual(_SCREAMING_SNAKE_CASE			,     _SCREAMING_SNAKE_CASE)