



from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel







@require_tf
class 							a_  :
						UpperCamelCase__      :						List[str]				       =MarianConfig
						UpperCamelCase__      :						Optional[int]				       ={}
						UpperCamelCase__      :						Any				       ="gelu"

						def __init__(    self	:int   ,						_lowercase	:str   ,						_lowercase	:List[Any]=13   ,						_lowercase	:Optional[Any]=7   ,						_lowercase	:Union[str, Any]=True   ,						_lowercase	:List[str]=False   ,						_lowercase	:Dict=99   ,						_lowercase	:List[str]=32   ,						_lowercase	:Tuple=2   ,						_lowercase	:str=4   ,						_lowercase	:Union[str, Any]=37   ,						_lowercase	:Tuple=0.1   ,						_lowercase	:Union[str, Any]=0.1   ,						_lowercase	:Dict=20   ,						_lowercase	:Optional[Any]=2   ,						_lowercase	:str=1   ,						_lowercase	:Any=0   ,						)		->					Tuple:
												UpperCAmelCase_     							=     parent
												UpperCAmelCase_     							=     batch_size
												UpperCAmelCase_     							=     seq_length
												UpperCAmelCase_     							=     is_training
												UpperCAmelCase_     							=     use_labels
												UpperCAmelCase_     							=     vocab_size
												UpperCAmelCase_     							=     hidden_size
												UpperCAmelCase_     							=     num_hidden_layers
												UpperCAmelCase_     							=     num_attention_heads
												UpperCAmelCase_     							=     intermediate_size

												UpperCAmelCase_     							=     hidden_dropout_prob
												UpperCAmelCase_     							=     attention_probs_dropout_prob
												UpperCAmelCase_     							=     max_position_embeddings
												UpperCAmelCase_     							=     eos_token_id
												UpperCAmelCase_     							=     pad_token_id
												UpperCAmelCase_     							=     bos_token_id

						def 	__a       (    self	:Optional[Any])		->					Dict:
												UpperCAmelCase_     							=     ids_tensor([self.batch_size, self.seq_length - 1]   ,						self.vocab_size)
												UpperCAmelCase_     							=     tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size)   ,						1)
												UpperCAmelCase_     							=     tf.concat([input_ids, eos_tensor]   ,						axis=1)

												UpperCAmelCase_     							=     ids_tensor([self.batch_size, self.seq_length]   ,						self.vocab_size)

												UpperCAmelCase_     							=     self.config_cls(
												    vocab_size=self.vocab_size   ,						d_model=self.hidden_size   ,						encoder_layers=self.num_hidden_layers   ,						decoder_layers=self.num_hidden_layers   ,						encoder_attention_heads=self.num_attention_heads   ,						decoder_attention_heads=self.num_attention_heads   ,						encoder_ffn_dim=self.intermediate_size   ,						decoder_ffn_dim=self.intermediate_size   ,						dropout=self.hidden_dropout_prob   ,						attention_dropout=self.attention_probs_dropout_prob   ,						max_position_embeddings=self.max_position_embeddings   ,						eos_token_ids=[2]   ,						bos_token_id=self.bos_token_id   ,						pad_token_id=self.pad_token_id   ,						decoder_start_token_id=self.pad_token_id   ,						**self.config_updates   ,						)
												UpperCAmelCase_     							=     prepare_marian_inputs_dict(_lowercase   ,						_lowercase   ,						_lowercase)
												return config, inputs_dict




						def 	__a       (    self	:Union[str, Any]   ,						_lowercase	:str   ,						_lowercase	:Any)		->					List[str]:
												UpperCAmelCase_     							=     TFMarianModel(config=_lowercase).get_decoder()
												UpperCAmelCase_     							=     inputs_dict['''input_ids''']

												UpperCAmelCase_     							=     input_ids[:1, :]
												UpperCAmelCase_     							=     inputs_dict['''attention_mask'''][:1, :]
												UpperCAmelCase_     							=     inputs_dict['''head_mask''']
												UpperCAmelCase_     							=     1

												# first forward pass
												UpperCAmelCase_     							=     model(_lowercase   ,						attention_mask=_lowercase   ,						head_mask=_lowercase   ,						use_cache=_lowercase)

												UpperCAmelCase_							,      UpperCAmelCase_     							=     outputs.to_tuple()

												# create hypothetical next token and extent to next_input_ids
												UpperCAmelCase_     							=     ids_tensor((self.batch_size, 3)   ,						config.vocab_size)
												UpperCAmelCase_     							=     tf.cast(ids_tensor((self.batch_size, 3)   ,						2)   ,						tf.inta)

												# append to next input_ids and
												UpperCAmelCase_     							=     tf.concat([input_ids, next_tokens]   ,						axis=-1)
												UpperCAmelCase_     							=     tf.concat([attention_mask, next_attn_mask]   ,						axis=-1)

												UpperCAmelCase_     							=     model(_lowercase   ,						attention_mask=_lowercase)[0]
												UpperCAmelCase_     							=     model(_lowercase   ,						attention_mask=_lowercase   ,						past_key_values=_lowercase)[0]

												self.parent.assertEqual(next_tokens.shape[1]   ,						output_from_past.shape[1])

												# select random slice
												UpperCAmelCase_     							=     int(ids_tensor((1,)   ,						output_from_past.shape[-1]))
												UpperCAmelCase_     							=     output_from_no_past[:, -3:, random_slice_idx]
												UpperCAmelCase_     							=     output_from_past[:, :, random_slice_idx]

												# test that outputs are equal for slice
												tf.debugging.assert_near(_lowercase   ,						_lowercase   ,						rtol=1E-3)







def 				A			(		__UpperCAmelCase							,					__UpperCAmelCase							,					__UpperCAmelCase							,					__UpperCAmelCase=None							,					__UpperCAmelCase=None							,					__UpperCAmelCase=None							,					__UpperCAmelCase=None							,					__UpperCAmelCase=None							,					)       ->       Union[str, Any]:



						'''simple docstring'''






						if attention_mask is None:
												UpperCAmelCase_     							=     tf.cast(tf.math.not_equal(__UpperCAmelCase							,					config.pad_token_id	)							,					tf.inta	)
						if decoder_attention_mask is None:
												UpperCAmelCase_     							=     tf.concat(
												    [
												        tf.ones(decoder_input_ids[:, :1].shape							,					dtype=tf.inta	),
												        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]							,					config.pad_token_id	)							,					tf.inta	),
												    ]							,					axis=-1							,					)
						if head_mask is None:
												UpperCAmelCase_     							=     tf.ones((config.encoder_layers, config.encoder_attention_heads)	)
						if decoder_head_mask is None:
												UpperCAmelCase_     							=     tf.ones((config.decoder_layers, config.decoder_attention_heads)	)
						if cross_attn_head_mask is None:
												UpperCAmelCase_     							=     tf.ones((config.decoder_layers, config.decoder_attention_heads)	)
						return {
						    "input_ids": input_ids,
						    "decoder_input_ids": decoder_input_ids,
						    "attention_mask": attention_mask,
						    "decoder_attention_mask": decoder_attention_mask,
						    "head_mask": head_mask,
						    "decoder_head_mask": decoder_head_mask,
						    "cross_attn_head_mask": cross_attn_head_mask,
						}







@require_tf
class 							a_  (		_snake_case					, _snake_case					, unittest.TestCase				):
						UpperCamelCase__      :						int				       =(TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
						UpperCamelCase__      :						Union[str, Any]				       =(TFMarianMTModel,) if is_tf_available() else ()
						UpperCamelCase__      :						List[Any]				       =(
						    {
						        "conversational": TFMarianMTModel,
						        "feature-extraction": TFMarianModel,
						        "summarization": TFMarianMTModel,
						        "text2text-generation": TFMarianMTModel,
						        "translation": TFMarianMTModel,
						    }
						    if is_tf_available()
						    else {}
						)
						UpperCamelCase__      :						List[Any]				       =True
						UpperCamelCase__      :						Union[str, Any]				       =False
						UpperCamelCase__      :						str				       =False

						def 	__a       (    self	:Union[str, Any])		->					List[Any]:
												UpperCAmelCase_     							=     TFMarianModelTester(self)
												UpperCAmelCase_     							=     ConfigTester(self   ,						config_class=_lowercase)

						def 	__a       (    self	:List[str])		->					Optional[Any]:
												self.config_tester.run_common_tests()




						def 	__a       (    self	:Optional[Any])		->					Optional[int]:
												UpperCAmelCase_     							=     self.model_tester.prepare_config_and_inputs_for_common()
												self.model_tester.check_decoder_model_past_large_inputs(*_lowercase)







@require_tf
class 							a_  (		unittest.TestCase				):
						UpperCamelCase__      :						str				       =10_00  # show more chars for failing integration tests

						@classmethod
						def 	__a       (    cls	:Optional[Any])		->					None:
												UpperCAmelCase_     							=     f"Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"
												return cls

						@cached_property
						def 	__a       (    self	:Tuple)		->					MarianTokenizer:
												return AutoTokenizer.from_pretrained(self.model_name)

						@property
						def 	__a       (    self	:Optional[int])		->					int:
												return self.tokenizer.eos_token_id

						@cached_property
						def 	__a       (    self	:Dict)		->					str:
												warnings.simplefilter('''error''')
												UpperCAmelCase_     							=     TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name)
												assert isinstance(_lowercase   ,						_lowercase)
												UpperCAmelCase_     							=     model.config
												self.assertListEqual(c.bad_words_ids   ,						[[c.pad_token_id]])
												self.assertEqual(c.max_length   ,						512)
												self.assertEqual(c.decoder_start_token_id   ,						c.pad_token_id)
												return model

						def 	__a       (    self	:Dict   ,						**_lowercase	:str)		->					Union[str, Any]:
												UpperCAmelCase_     							=     self.translate_src_text(**_lowercase)
												self.assertListEqual(self.expected_text   ,						_lowercase)




						def 	__a       (    self	:Union[str, Any]   ,						**_lowercase	:List[str])		->					Dict:
												UpperCAmelCase_     							=     self.tokenizer(self.src_text   ,						**_lowercase   ,						padding=_lowercase   ,						return_tensors='''tf''')
												UpperCAmelCase_     							=     self.model.generate(
												    model_inputs.input_ids   ,						attention_mask=model_inputs.attention_mask   ,						num_beams=2   ,						max_length=128)
												UpperCAmelCase_     							=     self.tokenizer.batch_decode(generated_ids.numpy()   ,						skip_special_tokens=_lowercase)
												return generated_words







@require_sentencepiece
@require_tokenizers
@require_tf
class 							a_  (		_snake_case				):
						UpperCamelCase__      :						str				       ="mt"
						UpperCamelCase__      :						Any				       ="en"
						UpperCamelCase__      :						int				       =["Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."]
						UpperCamelCase__      :						str				       =["Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."]

						@unittest.skip('''Skipping until #12647 is resolved.''')
						@slow
						def 	__a       (    self	:Dict)		->					Union[str, Any]:
												self._assert_generated_batch_equal_expected()







@require_sentencepiece
@require_tokenizers
@require_tf
class 							a_  (		_snake_case				):
						UpperCamelCase__      :						int				       ="en"
						UpperCamelCase__      :						Union[str, Any]				       ="zh"
						UpperCamelCase__      :						List[Any]				       =["My name is Wolfgang and I live in Berlin"]
						UpperCamelCase__      :						Any				       =["我叫沃尔夫冈 我住在柏林"]

						@unittest.skip('''Skipping until #12647 is resolved.''')
						@slow
						def 	__a       (    self	:Any)		->					List[Any]:
												self._assert_generated_batch_equal_expected()







@require_sentencepiece
@require_tokenizers
@require_tf
class 							a_  (		_snake_case				):
						UpperCamelCase__      :						int				       ="en"
						UpperCamelCase__      :						List[str]				       ="ROMANCE"
						UpperCamelCase__      :						Optional[int]				       =[
						    ">>fr<< Don't spend so much time watching TV.",
						    ">>pt<< Your message has been sent.",
						    ">>es<< He's two years older than me.",
						]
						UpperCamelCase__      :						Tuple				       =[
						    "Ne passez pas autant de temps à regarder la télé.",
						    "A sua mensagem foi enviada.",
						    "Es dos años más viejo que yo.",
						]

						@unittest.skip('''Skipping until #12647 is resolved.''')
						@slow
						def 	__a       (    self	:Optional[int])		->					Optional[int]:
												self._assert_generated_batch_equal_expected()




						@unittest.skip('''Skipping until #12647 is resolved.''')
						@slow
						def 	__a       (    self	:str)		->					Union[str, Any]:
												UpperCAmelCase_     							=     TranslationPipeline(self.model   ,						self.tokenizer   ,						framework='''tf''')
												UpperCAmelCase_     							=     pipeline(self.src_text)
												self.assertEqual(self.expected_text   ,						[x['''translation_text'''] for x in output])







'''simple docstring'''



import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Image
from datasets.packaged_modules.text.text import Text

from ..utils import require_pil
@pytest.fixture
def       lowerCAmelCase_		( SCREAMING_SNAKE_CASE_  )   ->   Union[str, Any]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE          = tmp_path / """text.txt"""
   _SCREAMING_SNAKE_CASE          = textwrap.dedent(
       """\
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.

        Second paragraph:
        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
        Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
        Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
        Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
        """  )
   with open(SCREAMING_SNAKE_CASE_			,					"""w"""			,					encoding="""utf-8"""  ) as f:
      f.write(SCREAMING_SNAKE_CASE_  )
   return str(SCREAMING_SNAKE_CASE_  )
@pytest.fixture
def       lowerCAmelCase_		( SCREAMING_SNAKE_CASE_			,					SCREAMING_SNAKE_CASE_  )   ->   Optional[int]:




   """simple docstring"""
   _SCREAMING_SNAKE_CASE          = tmp_path / """text_with_image.txt"""
   with open(SCREAMING_SNAKE_CASE_			,					"""w"""			,					encoding="""utf-8"""  ) as f:
      f.write(SCREAMING_SNAKE_CASE_  )
   return str(SCREAMING_SNAKE_CASE_  )
@pytest.mark.parametrize("""keep_linebreaks"""			,					[True, False]  )
def       lowerCAmelCase_		( SCREAMING_SNAKE_CASE_			,					SCREAMING_SNAKE_CASE_  )   ->   int:




   """simple docstring"""
   with open(SCREAMING_SNAKE_CASE_			,					encoding="""utf-8"""  ) as f:
      _SCREAMING_SNAKE_CASE          = f.read().splitlines(keepends=SCREAMING_SNAKE_CASE_  )
   _SCREAMING_SNAKE_CASE          = Text(keep_linebreaks=SCREAMING_SNAKE_CASE_			,					encoding="""utf-8"""  )
   _SCREAMING_SNAKE_CASE          = text._generate_tables([[text_file]]  )
   _SCREAMING_SNAKE_CASE          = pa.concat_tables([table for _, table in generator]  ).to_pydict()["""text"""]
   assert generated_content == expected_content
@require_pil
def       lowerCAmelCase_		( SCREAMING_SNAKE_CASE_  )   ->   List[str]:




   """simple docstring"""
   with open(SCREAMING_SNAKE_CASE_			,					encoding="""utf-8"""  ) as f:
      _SCREAMING_SNAKE_CASE          = f.read().splitlines()[0]
   _SCREAMING_SNAKE_CASE          = Text(encoding="""utf-8"""			,					features=Features({"""image""": Image()}  )  )
   _SCREAMING_SNAKE_CASE          = text._generate_tables([[text_file_with_image]]  )
   _SCREAMING_SNAKE_CASE          = pa.concat_tables([table for _, table in generator]  )
   assert pa_table.schema.field("""image"""  ).type == Image()()
   _SCREAMING_SNAKE_CASE          = pa_table.to_pydict()["""image"""]
   assert generated_content == [{"path": image_file, "bytes": None}]







@pytest.mark.parametrize("""sample_by"""			,					["""line""", """paragraph""", """document"""]  )
def       lowerCAmelCase_		( SCREAMING_SNAKE_CASE_			,					SCREAMING_SNAKE_CASE_  )   ->   List[Any]:




   """simple docstring"""
   with open(SCREAMING_SNAKE_CASE_			,					encoding="""utf-8"""  ) as f:
      _SCREAMING_SNAKE_CASE          = f.read()
   if sample_by == "line":
      _SCREAMING_SNAKE_CASE          = expected_content.splitlines()
   elif sample_by == "paragraph":
      _SCREAMING_SNAKE_CASE          = expected_content.split("""\n\n"""  )
   elif sample_by == "document":
      _SCREAMING_SNAKE_CASE          = [expected_content]
   _SCREAMING_SNAKE_CASE          = Text(sample_by=SCREAMING_SNAKE_CASE_			,					encoding="""utf-8"""			,					chunksize=1_00  )
   _SCREAMING_SNAKE_CASE          = text._generate_tables([[text_file]]  )
   _SCREAMING_SNAKE_CASE          = pa.concat_tables([table for _, table in generator]  ).to_pydict()["""text"""]
   assert generated_content == expected_content






