






import contextlib
import os
import sqlitea

import pytest

from datasets import Dataset, Features, Value
from datasets.io.sql import SqlDatasetReader, SqlDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases, require_sqlalchemy


def 				snake_case__							(							UpperCAmelCase							:  List[str]    ,			UpperCAmelCase							:  List[Any]       ):
    assert isinstance(UpperCAmelCase    ,			UpperCAmelCase       )
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
        assert dataset.features[feature].dtype == expected_dtype


@require_sqlalchemy
@pytest.mark.parametrize("keep_in_memory"    ,			[False, True]       )
def 				snake_case__							(							UpperCAmelCase							:  Optional[Any]    ,			UpperCAmelCase							:  List[str]    ,			UpperCAmelCase							:  int    ,			UpperCAmelCase							:  List[str]       ):
    lowerCAmelCase__   :List[Any]      							=				tmp_path / "cache"
    lowerCAmelCase__   :Any      							=				{"col_1": "string", "col_2": "int64", "col_3": "float64"}
    with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
        lowerCAmelCase__   :List[str]      							=				SqlDatasetReader(
            "dataset"    ,			"sqlite:///" + sqlite_path    ,			cache_dir=UpperCAmelCase    ,			keep_in_memory=UpperCAmelCase       ).read()
    _check_sql_dataset(UpperCAmelCase    ,			UpperCAmelCase       )


@require_sqlalchemy
@pytest.mark.parametrize(
    "features"    ,			[
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]    ,			)
def 				snake_case__							(							UpperCAmelCase							:  Union[str, Any]    ,			UpperCAmelCase							:  Optional[int]    ,			UpperCAmelCase							:  str    ,			UpperCAmelCase							:  Dict       ):
    lowerCAmelCase__   :Optional[int]      							=				tmp_path / "cache"
    lowerCAmelCase__   :Dict      							=				{"col_1": "string", "col_2": "int64", "col_3": "float64"}
    lowerCAmelCase__   :Tuple      							=				features.copy() if features else default_expected_features
    lowerCAmelCase__   :Optional[int]      							=				(
        Features({feature: Value(UpperCAmelCase       ) for feature, dtype in features.items()}       ) if features is not None else None
    )
    lowerCAmelCase__   :int      							=				SqlDatasetReader("dataset"    ,			"sqlite:///" + sqlite_path    ,			features=UpperCAmelCase    ,			cache_dir=UpperCAmelCase       ).read()
    _check_sql_dataset(UpperCAmelCase    ,			UpperCAmelCase       )


def 				snake_case__							(							UpperCAmelCase							:  Any       ):
    with contextlib.closing(sqlitea.connect(UpperCAmelCase       )       ) as con:
        lowerCAmelCase__   :Tuple      							=				con.cursor()
        cur.execute("SELECT * FROM dataset"       )
        for row in cur:
            yield row


@require_sqlalchemy
def 				snake_case__							(							UpperCAmelCase							:  List[Any]    ,			UpperCAmelCase							:  Optional[int]    ,			UpperCAmelCase							:  List[Any]       ):
    lowerCAmelCase__   :int      							=				tmp_path / "cache"
    lowerCAmelCase__   :str      							=				os.path.join(UpperCAmelCase    ,			"tmp.sql"       )
    lowerCAmelCase__   :Dict      							=				SqlDatasetReader("dataset"    ,			"sqlite:///" + sqlite_path    ,			cache_dir=UpperCAmelCase       ).read()
    SqlDatasetWriter(UpperCAmelCase    ,			"dataset"    ,			"sqlite:///" + output_sqlite_path    ,			num_proc=1       ).write()

    lowerCAmelCase__   :List[Any]      							=				iter_sql_file(UpperCAmelCase       )
    lowerCAmelCase__   :Union[str, Any]      							=				iter_sql_file(UpperCAmelCase       )

    for rowa, rowa in zip(UpperCAmelCase    ,			UpperCAmelCase       ):
        assert rowa == rowa


@require_sqlalchemy
def 				snake_case__							(							UpperCAmelCase							:  Optional[Any]    ,			UpperCAmelCase							:  str    ,			UpperCAmelCase							:  Tuple       ):
    lowerCAmelCase__   :List[Any]      							=				tmp_path / "cache"
    lowerCAmelCase__   :Union[str, Any]      							=				os.path.join(UpperCAmelCase    ,			"tmp.sql"       )
    lowerCAmelCase__   :Optional[Any]      							=				SqlDatasetReader("dataset"    ,			"sqlite:///" + sqlite_path    ,			cache_dir=UpperCAmelCase       ).read()
    SqlDatasetWriter(UpperCAmelCase    ,			"dataset"    ,			"sqlite:///" + output_sqlite_path    ,			num_proc=2       ).write()

    lowerCAmelCase__   :List[str]      							=				iter_sql_file(UpperCAmelCase       )
    lowerCAmelCase__   :List[str]      							=				iter_sql_file(UpperCAmelCase       )

    for rowa, rowa in zip(UpperCAmelCase    ,			UpperCAmelCase       ):
        assert rowa == rowa


@require_sqlalchemy
def 				snake_case__							(							UpperCAmelCase							:  Union[str, Any]    ,			UpperCAmelCase							:  Any    ,			UpperCAmelCase							:  int       ):
    lowerCAmelCase__   :int      							=				tmp_path / "cache"
    lowerCAmelCase__   :int      							=				os.path.join(UpperCAmelCase    ,			"tmp.sql"       )
    lowerCAmelCase__   :Tuple      							=				SqlDatasetReader("dataset"    ,			"sqlite:///" + sqlite_path    ,			cache_dir=UpperCAmelCase       ).read()
    with pytest.raises(UpperCAmelCase       ):
        SqlDatasetWriter(UpperCAmelCase    ,			"dataset"    ,			"sqlite:///" + output_sqlite_path    ,			num_proc=0       ).write()









from __future__ import annotations

import unittest

from transformers import AutoTokenizer, MBartConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import TFAutoModelForSeqaSeqLM, TFMBartForConditionalGeneration, TFMBartModel







@require_tf
class 	_UpperCAmelCase  :




      """simple docstring"""



      A          =    MBartConfig
      A          =    {}
      A          =    '''gelu'''
      def __init__(   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase=13    ,	_lowerCAmelCase=7    ,	_lowerCAmelCase=True    ,	_lowerCAmelCase=False    ,	_lowerCAmelCase=99    ,	_lowerCAmelCase=32    ,	_lowerCAmelCase=2    ,	_lowerCAmelCase=4    ,	_lowerCAmelCase=37    ,	_lowerCAmelCase=0.1    ,	_lowerCAmelCase=0.1    ,	_lowerCAmelCase=20    ,	_lowerCAmelCase=2    ,	_lowerCAmelCase=1    ,	_lowerCAmelCase=0    ,	):



          '''simple docstring'''
          lowerCAmelCase__   :Optional[int]      							=				parent
          lowerCAmelCase__   :Dict      							=				batch_size
          lowerCAmelCase__   :Optional[int]      							=				seq_length
          lowerCAmelCase__   :Any      							=				is_training
          lowerCAmelCase__   :List[Any]      							=				use_labels
          lowerCAmelCase__   :Optional[int]      							=				vocab_size
          lowerCAmelCase__   :Optional[int]      							=				hidden_size
          lowerCAmelCase__   :Optional[Any]      							=				num_hidden_layers
          lowerCAmelCase__   :List[str]      							=				num_attention_heads
          lowerCAmelCase__   :List[str]      							=				intermediate_size
          lowerCAmelCase__   :List[str]      							=				hidden_dropout_prob
          lowerCAmelCase__   :Optional[Any]      							=				attention_probs_dropout_prob
          lowerCAmelCase__   :Tuple      							=				max_position_embeddings
          lowerCAmelCase__   :Optional[Any]      							=				eos_token_id
          lowerCAmelCase__   :int      							=				pad_token_id
          lowerCAmelCase__   :Any      							=				bos_token_id
      def 			snake_case_      (   self		):



          '''simple docstring'''
          lowerCAmelCase__   :Tuple      							=				ids_tensor([self.batch_size, self.seq_length - 1]    ,	self.vocab_size		)
          lowerCAmelCase__   :List[Any]      							=				tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size		)    ,	1		)
          lowerCAmelCase__   :Union[str, Any]      							=				tf.concat([input_ids, eos_tensor]    ,	axis=1		)

          lowerCAmelCase__   :str      							=				ids_tensor([self.batch_size, self.seq_length]    ,	self.vocab_size		)

          lowerCAmelCase__   :Dict      							=				self.config_cls(
              vocab_size=self.vocab_size    ,	d_model=self.hidden_size    ,	encoder_layers=self.num_hidden_layers    ,	decoder_layers=self.num_hidden_layers    ,	encoder_attention_heads=self.num_attention_heads    ,	decoder_attention_heads=self.num_attention_heads    ,	encoder_ffn_dim=self.intermediate_size    ,	decoder_ffn_dim=self.intermediate_size    ,	dropout=self.hidden_dropout_prob    ,	attention_dropout=self.attention_probs_dropout_prob    ,	max_position_embeddings=self.max_position_embeddings    ,	eos_token_ids=[2]    ,	bos_token_id=self.bos_token_id    ,	pad_token_id=self.pad_token_id    ,	decoder_start_token_id=self.pad_token_id    ,	**self.config_updates    ,	)
          lowerCAmelCase__   :Union[str, Any]      							=				prepare_mbart_inputs_dict(_lowerCAmelCase    ,	_lowerCAmelCase    ,	_lowerCAmelCase		)
          return config, inputs_dict



      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase		):



          '''simple docstring'''
          lowerCAmelCase__   :int      							=				TFMBartModel(config=_lowerCAmelCase		).get_decoder()
          lowerCAmelCase__   :Tuple      							=				inputs_dict["input_ids"]

          lowerCAmelCase__   :Dict      							=				input_ids[:1, :]
          lowerCAmelCase__   :Dict      							=				inputs_dict["attention_mask"][:1, :]
          lowerCAmelCase__   :List[str]      							=				inputs_dict["head_mask"]
          lowerCAmelCase__   :Optional[Any]      							=				1

          # first forward pass
          lowerCAmelCase__   :List[Any]      							=				model(_lowerCAmelCase    ,	attention_mask=_lowerCAmelCase    ,	head_mask=_lowerCAmelCase    ,	use_cache=_lowerCAmelCase		)

          lowerCAmelCase__							,lowerCAmelCase__   :Any      							=				outputs.to_tuple()
          lowerCAmelCase__   :Tuple      							=				past_key_values[1]





def 				snake_case__							(							UpperCAmelCase							:  Union[str, Any]    ,			UpperCAmelCase							:  Optional[Any]    ,			UpperCAmelCase							:  Optional[int]    ,			UpperCAmelCase							:  int=None    ,			UpperCAmelCase							:  Optional[Any]=None    ,			UpperCAmelCase							:  Any=None    ,			UpperCAmelCase							:  Dict=None    ,			UpperCAmelCase							:  Dict=None    ,			):
    if attention_mask is None:
        lowerCAmelCase__   :str      							=				tf.cast(tf.math.not_equal(UpperCAmelCase    ,			config.pad_token_id       )    ,			tf.inta       )
    if decoder_attention_mask is None:
        lowerCAmelCase__   :int      							=				tf.concat(
            [
                tf.ones(decoder_input_ids[:, :1].shape    ,			dtype=tf.inta       ),
                tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]    ,			config.pad_token_id       )    ,			tf.inta       ),
            ]    ,			axis=-1    ,			)
    if head_mask is None:
        lowerCAmelCase__   :str      							=				tf.ones((config.encoder_layers, config.encoder_attention_heads)       )
    if decoder_head_mask is None:
        lowerCAmelCase__   :Tuple      							=				tf.ones((config.decoder_layers, config.decoder_attention_heads)       )
    if cross_attn_head_mask is None:
        lowerCAmelCase__   :Optional[Any]      							=				tf.ones((config.decoder_layers, config.decoder_attention_heads)       )
    return {
        "input_ids": input_ids,
        "decoder_input_ids": decoder_input_ids,
        "attention_mask": attention_mask,
        "decoder_attention_mask": decoder_attention_mask,
        "head_mask": head_mask,
        "decoder_head_mask": decoder_head_mask,
        "cross_attn_head_mask": cross_attn_head_mask,
    }







@require_tf
class 	_UpperCAmelCase  (  _A				,							_A				,							unittest.TestCase		):




      """simple docstring"""



      A          =    (TFMBartForConditionalGeneration, TFMBartModel) if is_tf_available() else ()
      A          =    (TFMBartForConditionalGeneration,) if is_tf_available() else ()
      A          =    (
          {
              '''conversational''': TFMBartForConditionalGeneration,
              '''feature-extraction''': TFMBartModel,
              '''summarization''': TFMBartForConditionalGeneration,
              '''text2text-generation''': TFMBartForConditionalGeneration,
              '''translation''': TFMBartForConditionalGeneration,
          }
          if is_tf_available()
          else {}
      )
      A          =    True
      A          =    False
      A          =    False
      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase    ,	_lowerCAmelCase    ,	_lowerCAmelCase    ,	_lowerCAmelCase		):



          '''simple docstring'''
          if pipeline_test_casse_name != "FeatureExtractionPipelineTests":
              # Exception encountered when calling layer '...'
              return True

          return False
      def 			snake_case_      (   self		):



          '''simple docstring'''
          lowerCAmelCase__   :List[str]      							=				TFMBartModelTester(self		)
          lowerCAmelCase__   :Union[str, Any]      							=				ConfigTester(self    ,	config_class=_lowerCAmelCase		)
      def 			snake_case_      (   self		):



          '''simple docstring'''
          self.config_tester.run_common_tests()



      def 			snake_case_      (   self		):



          '''simple docstring'''
          lowerCAmelCase__   :Tuple      							=				self.model_tester.prepare_config_and_inputs_for_common()
          self.model_tester.check_decoder_model_past_large_inputs(*_lowerCAmelCase		)







@require_sentencepiece
@require_tokenizers
@require_tf
class 	_UpperCAmelCase  (  unittest.TestCase		):




      """simple docstring"""



      A          =    [
          ''' UN Chief Says There Is No Military Solution in Syria''',
      ]
      A          =    [
          '''Şeful ONU declară că nu există o soluţie militară în Siria''',
      ]
      A          =    '''facebook/mbart-large-en-ro'''
      @cached_property
      def 			snake_case_      (   self		):



          '''simple docstring'''
          return AutoTokenizer.from_pretrained(self.model_name		)
      @cached_property
      def 			snake_case_      (   self		):



          '''simple docstring'''
          lowerCAmelCase__   :Union[str, Any]      							=				TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name		)
          return model
      def 			snake_case_      (   self    ,	**_lowerCAmelCase		):



          '''simple docstring'''
          lowerCAmelCase__   :Tuple      							=				self.translate_src_text(**_lowerCAmelCase		)
          self.assertListEqual(self.expected_text    ,	_lowerCAmelCase		)
      def 			snake_case_      (   self    ,	**_lowerCAmelCase		):



          '''simple docstring'''
          lowerCAmelCase__   :Tuple      							=				self.tokenizer(self.src_text    ,	**_lowerCAmelCase    ,	return_tensors="tf"		)
          lowerCAmelCase__   :str      							=				self.model.generate(
              model_inputs.input_ids    ,	attention_mask=model_inputs.attention_mask    ,	num_beams=2		)
          lowerCAmelCase__   :int      							=				self.tokenizer.batch_decode(_lowerCAmelCase    ,	skip_special_tokens=_lowerCAmelCase		)
          return generated_words



      @slow
      def 			snake_case_      (   self		):



          '''simple docstring'''
          self._assert_generated_batch_equal_expected()



