


'''simple docstring'''

from __future__ import annotations

import unittest

from transformers import AutoTokenizer, MBartConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import TFAutoModelForSeqaSeqLM, TFMBartForConditionalGeneration, TFMBartModel





@require_tf
class 	snake_case       :





  """simple docstring"""




  _lowerCamelCase						       =    MBartConfig
  _lowerCamelCase						       =    {}
  _lowerCamelCase						       =    "gelu"






  def __init__(       self     ,    UpperCamelCase     ,    UpperCamelCase=13     ,    UpperCamelCase=7     ,    UpperCamelCase=True     ,    UpperCamelCase=False     ,    UpperCamelCase=99     ,    UpperCamelCase=32     ,    UpperCamelCase=2     ,    UpperCamelCase=4     ,    UpperCamelCase=37     ,    UpperCamelCase=0.1     ,    UpperCamelCase=0.1     ,    UpperCamelCase=20     ,    UpperCamelCase=2     ,    UpperCamelCase=1     ,    UpperCamelCase=0     ,    ):







   """simple docstring"""
   lowerCamelCase_						=    parent
   lowerCamelCase_						=    batch_size
   lowerCamelCase_						=    seq_length
   lowerCamelCase_						=    is_training
   lowerCamelCase_						=    use_labels
   lowerCamelCase_						=    vocab_size
   lowerCamelCase_						=    hidden_size
   lowerCamelCase_						=    num_hidden_layers
   lowerCamelCase_						=    num_attention_heads
   lowerCamelCase_						=    intermediate_size
   lowerCamelCase_						=    hidden_dropout_prob
   lowerCamelCase_						=    attention_probs_dropout_prob
   lowerCamelCase_						=    max_position_embeddings
   lowerCamelCase_						=    eos_token_id
   lowerCamelCase_						=    pad_token_id
   lowerCamelCase_						=    bos_token_id






  def     snake_case						(       self	):







   """simple docstring"""
   lowerCamelCase_						=    ids_tensor([self.batch_size, self.seq_length - 1]     ,    self.vocab_size	)
   lowerCamelCase_						=    tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size	)     ,    1	)
   lowerCamelCase_						=    tf.concat([input_ids, eos_tensor]     ,    axis=1	)

   lowerCamelCase_						=    ids_tensor([self.batch_size, self.seq_length]     ,    self.vocab_size	)

   lowerCamelCase_						=    self.config_cls(
       vocab_size=self.vocab_size     ,    d_model=self.hidden_size     ,    encoder_layers=self.num_hidden_layers     ,    decoder_layers=self.num_hidden_layers     ,    encoder_attention_heads=self.num_attention_heads     ,    decoder_attention_heads=self.num_attention_heads     ,    encoder_ffn_dim=self.intermediate_size     ,    decoder_ffn_dim=self.intermediate_size     ,    dropout=self.hidden_dropout_prob     ,    attention_dropout=self.attention_probs_dropout_prob     ,    max_position_embeddings=self.max_position_embeddings     ,    eos_token_ids=[2]     ,    bos_token_id=self.bos_token_id     ,    pad_token_id=self.pad_token_id     ,    decoder_start_token_id=self.pad_token_id     ,    **self.config_updates     ,    )
   lowerCamelCase_						=    prepare_mbart_inputs_dict(lowercase__     ,    lowercase__     ,    lowercase__	)
   return config, inputs_dict







  def     snake_case						(       self     ,    UpperCamelCase     ,    UpperCamelCase	):







   """simple docstring"""
   lowerCamelCase_						=    TFMBartModel(config=lowercase__	).get_decoder()
   lowerCamelCase_						=    inputs_dict["input_ids"]

   lowerCamelCase_						=    input_ids[:1, :]
   lowerCamelCase_						=    inputs_dict["attention_mask"][:1, :]
   lowerCamelCase_						=    inputs_dict["head_mask"]
   lowerCamelCase_						=    1

   # first forward pass
   lowerCamelCase_						=    model(lowercase__     ,    attention_mask=lowercase__     ,    head_mask=lowercase__     ,    use_cache=lowercase__	)

   lowerCamelCase_					,lowerCamelCase_						=    outputs.to_tuple()
   lowerCamelCase_						=    past_key_values[1]





def __snake_case   ( UpperCAmelCase_		:		Any   ,							UpperCAmelCase_		:		Union[str, Any]   ,							UpperCAmelCase_		:		Optional[Any]   ,							UpperCAmelCase_		:		str=None   ,							UpperCAmelCase_		:		str=None   ,							UpperCAmelCase_		:		Dict=None   ,							UpperCAmelCase_		:		Optional[Any]=None   ,							UpperCAmelCase_		:		Dict=None   ,							):
 if attention_mask is None:
  lowerCamelCase_						=    tf.cast(tf.math.not_equal(__A   ,							config.pad_token_id						)   ,							tf.inta						)
 if decoder_attention_mask is None:
  lowerCamelCase_						=    tf.concat(
      [
          tf.ones(decoder_input_ids[:, :1].shape   ,							dtype=tf.inta						),
          tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]   ,							config.pad_token_id						)   ,							tf.inta						),
      ]   ,							axis=-1   ,							)
 if head_mask is None:
  lowerCamelCase_						=    tf.ones((config.encoder_layers, config.encoder_attention_heads)						)
 if decoder_head_mask is None:
  lowerCamelCase_						=    tf.ones((config.decoder_layers, config.decoder_attention_heads)						)
 if cross_attn_head_mask is None:
  lowerCamelCase_						=    tf.ones((config.decoder_layers, config.decoder_attention_heads)						)
 return {
     "input_ids": input_ids,
     "decoder_input_ids": decoder_input_ids,
     "attention_mask": attention_mask,
     "decoder_attention_mask": decoder_attention_mask,
     "head_mask": head_mask,
     "decoder_head_mask": decoder_head_mask,
     "cross_attn_head_mask": cross_attn_head_mask,
 }





@require_tf
class 	snake_case       (       lowercase     ,		lowercase     ,		unittest.TestCase     ):





  """simple docstring"""




  _lowerCamelCase						       =    (TFMBartForConditionalGeneration, TFMBartModel) if is_tf_available() else ()
  _lowerCamelCase						       =    (TFMBartForConditionalGeneration,) if is_tf_available() else ()
  _lowerCamelCase						       =    (
      {
          "conversational": TFMBartForConditionalGeneration,
          "feature-extraction": TFMBartModel,
          "summarization": TFMBartForConditionalGeneration,
          "text2text-generation": TFMBartForConditionalGeneration,
          "translation": TFMBartForConditionalGeneration,
      }
      if is_tf_available()
      else {}
  )
  _lowerCamelCase						       =    True
  _lowerCamelCase						       =    False
  _lowerCamelCase						       =    False






  def     snake_case						(       self     ,    UpperCamelCase     ,    UpperCamelCase     ,    UpperCamelCase     ,    UpperCamelCase     ,    UpperCamelCase	):







   """simple docstring"""



   if pipeline_test_casse_name != "FeatureExtractionPipelineTests":
    # Exception encountered when calling layer '...'
    return True

   return False






  def     snake_case						(       self	):







   """simple docstring"""
   lowerCamelCase_						=    TFMBartModelTester(self	)
   lowerCamelCase_						=    ConfigTester(self     ,    config_class=lowercase__	)






  def     snake_case						(       self	):







   """simple docstring"""
   self.config_tester.run_common_tests()







  def     snake_case						(       self	):







   """simple docstring"""
   lowerCamelCase_						=    self.model_tester.prepare_config_and_inputs_for_common()
   self.model_tester.check_decoder_model_past_large_inputs(*lowercase__	)





@require_sentencepiece
@require_tokenizers
@require_tf
class 	snake_case       (       unittest.TestCase     ):





  """simple docstring"""




  _lowerCamelCase						       =    [
      " UN Chief Says There Is No Military Solution in Syria",
  ]
  _lowerCamelCase						       =    [
      "Şeful ONU declară că nu există o soluţie militară în Siria",
  ]
  _lowerCamelCase						       =    "facebook/mbart-large-en-ro"






  @cached_property
  def     snake_case						(       self	):







   """simple docstring"""
   return AutoTokenizer.from_pretrained(self.model_name	)






  @cached_property
  def     snake_case						(       self	):







   """simple docstring"""
   lowerCamelCase_						=    TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name	)
   return model






  def     snake_case						(       self     ,    **UpperCamelCase	):







   """simple docstring"""
   lowerCamelCase_						=    self.translate_src_text(**lowercase__	)
   self.assertListEqual(self.expected_text     ,    lowercase__	)






  def     snake_case						(       self     ,    **UpperCamelCase	):







   """simple docstring"""
   lowerCamelCase_						=    self.tokenizer(self.src_text     ,    **lowercase__     ,    return_tensors="tf"	)
   lowerCamelCase_						=    self.model.generate(
       model_inputs.input_ids     ,    attention_mask=model_inputs.attention_mask     ,    num_beams=2	)
   lowerCamelCase_						=    self.tokenizer.batch_decode(lowercase__     ,    skip_special_tokens=lowercase__	)
   return generated_words







  @slow
  def     snake_case						(       self	):







   """simple docstring"""
   self._assert_generated_batch_equal_expected()






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


_lowerCamelCase      :					List[str]					    =      pytest.mark.integration





@require_faiss
class        lowerCAmelCase__  (					__magic_name__			):



       '''simple docstring'''







       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             __A    =Dataset.from_dict({'''filename''': ['''my_name-train''' + '''_''' + str(lowercase__     ) for x in np.arange(3_0     ).tolist()]}     )
             return dset

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =self._create_dummy_dataset()
             __A    =dset.map(
                 lambda lowercase__ ,   lowercase__     : {"vecs": i * np.ones(5 ,   dtype=np.floataa     )} ,   with_indices=lowercase__ ,   keep_in_memory=lowercase__     )
             __A    =dset.add_faiss_index('''vecs''' ,   batch_size=1_0_0 ,   metric_type=faiss.METRIC_INNER_PRODUCT     )
             __A    ,		__A    =dset.get_nearest_examples('''vecs''' ,   np.ones(5 ,   dtype=np.floataa     )     )
             self.assertEqual(examples['''filename'''][0] ,   '''my_name-train_29'''     )
             dset.drop_index('''vecs'''     )

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =self._create_dummy_dataset()
             dset.add_faiss_index_from_external_arrays(
                 external_arrays=np.ones((3_0, 5)     ) * np.arange(3_0     ).reshape(-1 ,   1     ) ,   index_name='''vecs''' ,   batch_size=1_0_0 ,   metric_type=faiss.METRIC_INNER_PRODUCT ,   )
             __A    ,		__A    =dset.get_nearest_examples('''vecs''' ,   np.ones(5 ,   dtype=np.floataa     )     )
             self.assertEqual(examples['''filename'''][0] ,   '''my_name-train_29'''     )

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =self._create_dummy_dataset()
             dset.add_faiss_index_from_external_arrays(
                 external_arrays=np.ones((3_0, 5)     ) * np.arange(3_0     ).reshape(-1 ,   1     ) ,   index_name='''vecs''' ,   metric_type=faiss.METRIC_INNER_PRODUCT ,   )

             # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
             # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
             # see https://bugs.python.org/issue14243 and
             # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
             with tempfile.NamedTemporaryFile(delete=lowercase__     ) as tmp_file:
                   dset.save_faiss_index('''vecs''' ,   tmp_file.name     )
                   dset.load_faiss_index('''vecs2''' ,   tmp_file.name     )
             os.unlink(tmp_file.name     )

             __A    ,		__A    =dset.get_nearest_examples('''vecs2''' ,   np.ones(5 ,   dtype=np.floataa     )     )
             self.assertEqual(examples['''filename'''][0] ,   '''my_name-train_29'''     )

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             __A    =self._create_dummy_dataset()
             dset.add_faiss_index_from_external_arrays(
                 external_arrays=np.ones((3_0, 5)     ) * np.arange(3_0     ).reshape(-1 ,   1     ) ,   index_name='''vecs'''     )
             dset.drop_index('''vecs'''     )
             self.assertRaises(lowercase__ ,   partial(dset.get_nearest_examples ,   '''vecs2''' ,   np.ones(5 ,   dtype=np.floataa     )     )     )


       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             from elasticsearch import Elasticsearch

             __A    =self._create_dummy_dataset()
             with patch('''elasticsearch.Elasticsearch.search'''     ) as mocked_search, patch(
                 '''elasticsearch.client.IndicesClient.create'''     ) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''     ) as mocked_bulk:
                   __A    ={'''acknowledged''': True}
                   mocked_bulk.return_value([(True, None)] * 3_0     )
                   __A    ={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 2_9}]}}
                   __A    =Elasticsearch()

                   dset.add_elasticsearch_index('''filename''' ,   es_client=lowercase__     )
                   __A    ,		__A    =dset.get_nearest_examples('''filename''' ,   '''my_name-train_29'''     )
                   self.assertEqual(examples['''filename'''][0] ,   '''my_name-train_29'''     )





@require_faiss
class        lowerCAmelCase__  (					__magic_name__			):



       '''simple docstring'''







       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT     )

             # add vectors
             index.add_vectors(np.eye(5 ,   dtype=np.floataa     )     )
             self.assertIsNotNone(index.faiss_index     )
             self.assertEqual(index.faiss_index.ntotal ,   5     )
             index.add_vectors(np.zeros((5, 5) ,   dtype=np.floataa     )     )
             self.assertEqual(index.faiss_index.ntotal ,   1_0     )

             # single query
             __A    =np.zeros(5 ,   dtype=np.floataa     )
             __A    =1
             __A    ,		__A    =index.search(lowercase__     )
             self.assertRaises(lowercase__ ,   index.search ,   query.reshape(-1 ,   1     )     )
             self.assertGreater(scores[0] ,   0     )
             self.assertEqual(indices[0] ,   1     )

             # batched queries
             __A    =np.eye(5 ,   dtype=np.floataa     )[::-1]
             __A    ,		__A    =index.search_batch(lowercase__     )
             self.assertRaises(lowercase__ ,   index.search_batch ,   queries[0]     )
             __A    =[scores[0] for scores in total_scores]
             __A    =[indices[0] for indices in total_indices]
             self.assertGreater(np.min(lowercase__     ) ,   0     )
             self.assertListEqual([4, 3, 2, 1, 0] ,   lowercase__     )

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =FaissIndex(string_factory='''Flat'''     )
             index.add_vectors(np.eye(5 ,   dtype=np.floataa     )     )
             self.assertIsInstance(index.faiss_index ,   faiss.IndexFlat     )
             __A    =FaissIndex(string_factory='''LSH'''     )
             index.add_vectors(np.eye(5 ,   dtype=np.floataa     )     )
             self.assertIsInstance(index.faiss_index ,   faiss.IndexLSH     )
             with self.assertRaises(lowercase__     ):
                   __A    =FaissIndex(string_factory='''Flat''' ,   custom_index=faiss.IndexFlat(5     )     )

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =faiss.IndexFlat(5     )
             __A    =FaissIndex(custom_index=lowercase__     )
             index.add_vectors(np.eye(5 ,   dtype=np.floataa     )     )
             self.assertIsInstance(index.faiss_index ,   faiss.IndexFlat     )


       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             import faiss

             __A    =FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT     )
             index.add_vectors(np.eye(5 ,   dtype=np.floataa     )     )

             # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
             # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
             # see https://bugs.python.org/issue14243 and
             # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
             with tempfile.NamedTemporaryFile(delete=lowercase__     ) as tmp_file:
                   index.save(tmp_file.name     )
                   __A    =FaissIndex.load(tmp_file.name     )
             os.unlink(tmp_file.name     )

             __A    =np.zeros(5 ,   dtype=np.floataa     )
             __A    =1
             __A    ,		__A    =index.search(lowercase__     )
             self.assertGreater(scores[0] ,   0     )
             self.assertEqual(indices[0] ,   1     )





@require_faiss
def      A__					(			__A  :						Dict			)					->List[Any]:
      import faiss

      __A    =FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT			)
      index.add_vectors(np.eye(5     ,				dtype=np.floataa			)			)

      __A    ='''index.faiss'''
      __A    =F'''mock://{index_name}'''
      index.save(__A     ,				storage_options=mockfs.storage_options			)
      __A    =FaissIndex.load(__A     ,				storage_options=mockfs.storage_options			)

      __A    =np.zeros(5     ,				dtype=np.floataa			)
      __A    =1
      __A    ,		__A    =index.search(__A			)
      assert scores[0] > 0
      assert indices[0] == 1





@require_elasticsearch
class        lowerCAmelCase__  (					__magic_name__			):



       '''simple docstring'''







       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             from elasticsearch import Elasticsearch

             with patch('''elasticsearch.Elasticsearch.search'''     ) as mocked_search, patch(
                 '''elasticsearch.client.IndicesClient.create'''     ) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''     ) as mocked_bulk:
                   __A    =Elasticsearch()
                   __A    ={'''acknowledged''': True}
                   __A    =ElasticSearchIndex(es_client=lowercase__     )
                   mocked_bulk.return_value([(True, None)] * 3     )
                   index.add_documents(['''foo''', '''bar''', '''foobar''']     )

                   # single query
                   __A    ='''foo'''
                   __A    ={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
                   __A    ,		__A    =index.search(lowercase__     )
                   self.assertEqual(scores[0] ,   1     )
                   self.assertEqual(indices[0] ,   0     )

                   # single query with timeout
                   __A    ='''foo'''
                   __A    ={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
                   __A    ,		__A    =index.search(lowercase__ ,   request_timeout=3_0     )
                   self.assertEqual(scores[0] ,   1     )
                   self.assertEqual(indices[0] ,   0     )

                   # batched queries
                   __A    =['''foo''', '''bar''', '''foobar''']
                   __A    ={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
                   __A    ,		__A    =index.search_batch(lowercase__     )
                   __A    =[scores[0] for scores in total_scores]
                   __A    =[indices[0] for indices in total_indices]
                   self.assertGreater(np.min(lowercase__     ) ,   0     )
                   self.assertListEqual([1, 1, 1] ,   lowercase__     )

                   # batched queries with timeout
                   __A    =['''foo''', '''bar''', '''foobar''']
                   __A    ={'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
                   __A    ,		__A    =index.search_batch(lowercase__ ,   request_timeout=3_0     )
                   __A    =[scores[0] for scores in total_scores]
                   __A    =[indices[0] for indices in total_indices]
                   self.assertGreater(np.min(lowercase__     ) ,   0     )
                   self.assertListEqual([1, 1, 1] ,   lowercase__     )


