





'''simple docstring'''


import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


a__			:					int       						=						pytest.mark.integration






@require_faiss
class     lowerCAmelCase__ (				UpperCAmelCase_    ):

       '''simple docstring'''

       def 			__snake_case      ( self					:				Any						):
              UpperCAmelCase  =						Dataset.from_dict({'''filename''': ['''my_name-train''' + '''_''' + str(a__						) for x in np.arange(30						).tolist()]}						)
              return dset

       def 			__snake_case      ( self					:				List[str]						):
              import faiss

              UpperCAmelCase  =						self._create_dummy_dataset()
              UpperCAmelCase  =						dset.map(
                  lambda a__       ,     a__						: {"vecs": i * np.ones(5       ,     dtype=np.floataa						)}       ,     with_indices=a__       ,     keep_in_memory=a__						)
              UpperCAmelCase  =						dset.add_faiss_index('''vecs'''       ,     batch_size=100       ,     metric_type=faiss.METRIC_INNER_PRODUCT						)
              UpperCAmelCase,					UpperCAmelCase  =						dset.get_nearest_examples('''vecs'''       ,     np.ones(5       ,     dtype=np.floataa						)						)
              self.assertEqual(examples['''filename'''][0]       ,     '''my_name-train_29'''						)
              dset.drop_index('''vecs'''						)

       def 			__snake_case      ( self					:				str						):
              import faiss

              UpperCAmelCase  =						self._create_dummy_dataset()
              dset.add_faiss_index_from_external_arrays(
                  external_arrays=np.ones((30, 5)						) * np.arange(30						).reshape(-1       ,     1						)       ,     index_name='''vecs'''       ,     batch_size=100       ,     metric_type=faiss.METRIC_INNER_PRODUCT       ,     )
              UpperCAmelCase,					UpperCAmelCase  =						dset.get_nearest_examples('''vecs'''       ,     np.ones(5       ,     dtype=np.floataa						)						)
              self.assertEqual(examples['''filename'''][0]       ,     '''my_name-train_29'''						)

       def 			__snake_case      ( self					:				List[Any]						):
              import faiss

              UpperCAmelCase  =						self._create_dummy_dataset()
              dset.add_faiss_index_from_external_arrays(
                  external_arrays=np.ones((30, 5)						) * np.arange(30						).reshape(-1       ,     1						)       ,     index_name='''vecs'''       ,     metric_type=faiss.METRIC_INNER_PRODUCT       ,     )

              # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
              # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
              # see https://bugs.python.org/issue14243 and
              # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
              with tempfile.NamedTemporaryFile(delete=a__						) as tmp_file:
                     dset.save_faiss_index('''vecs'''       ,     tmp_file.name						)
                     dset.load_faiss_index('''vecs2'''       ,     tmp_file.name						)
              os.unlink(tmp_file.name						)

              UpperCAmelCase,					UpperCAmelCase  =						dset.get_nearest_examples('''vecs2'''       ,     np.ones(5       ,     dtype=np.floataa						)						)
              self.assertEqual(examples['''filename'''][0]       ,     '''my_name-train_29'''						)

       def 			__snake_case      ( self					:				Tuple						):
              UpperCAmelCase  =						self._create_dummy_dataset()
              dset.add_faiss_index_from_external_arrays(
                  external_arrays=np.ones((30, 5)						) * np.arange(30						).reshape(-1       ,     1						)       ,     index_name='''vecs'''						)
              dset.drop_index('''vecs'''						)
              self.assertRaises(a__       ,     partial(dset.get_nearest_examples       ,     '''vecs2'''       ,     np.ones(5       ,     dtype=np.floataa						)						)						)



       def 			__snake_case      ( self					:				int						):
              from elasticsearch import Elasticsearch

              UpperCAmelCase  =						self._create_dummy_dataset()
              with patch('''elasticsearch.Elasticsearch.search'''						) as mocked_search, patch(
                  '''elasticsearch.client.IndicesClient.create'''						) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''						) as mocked_bulk:
                     UpperCAmelCase  =						{'''acknowledged''': True}
                     mocked_bulk.return_value([(True, None)] * 30						)
                     UpperCAmelCase  =						{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 29}]}}
                     UpperCAmelCase  =						Elasticsearch()

                     dset.add_elasticsearch_index('''filename'''       ,     es_client=a__						)
                     UpperCAmelCase,					UpperCAmelCase  =						dset.get_nearest_examples('''filename'''       ,     '''my_name-train_29'''						)
                     self.assertEqual(examples['''filename'''][0]       ,     '''my_name-train_29'''						)






@require_faiss
class     lowerCAmelCase__ (				UpperCAmelCase_    ):

       '''simple docstring'''

       def 			__snake_case      ( self					:				Optional[Any]						):
              import faiss

              UpperCAmelCase  =						FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT						)

              # add vectors
              index.add_vectors(np.eye(5       ,     dtype=np.floataa						)						)
              self.assertIsNotNone(index.faiss_index						)
              self.assertEqual(index.faiss_index.ntotal       ,     5						)
              index.add_vectors(np.zeros((5, 5)       ,     dtype=np.floataa						)						)
              self.assertEqual(index.faiss_index.ntotal       ,     10						)

              # single query
              UpperCAmelCase  =						np.zeros(5       ,     dtype=np.floataa						)
              UpperCAmelCase  =						1
              UpperCAmelCase,					UpperCAmelCase  =						index.search(a__						)
              self.assertRaises(a__       ,     index.search       ,     query.reshape(-1       ,     1						)						)
              self.assertGreater(scores[0]       ,     0						)
              self.assertEqual(indices[0]       ,     1						)

              # batched queries
              UpperCAmelCase  =						np.eye(5       ,     dtype=np.floataa						)[::-1]
              UpperCAmelCase,					UpperCAmelCase  =						index.search_batch(a__						)
              self.assertRaises(a__       ,     index.search_batch       ,     queries[0]						)
              UpperCAmelCase  =						[scores[0] for scores in total_scores]
              UpperCAmelCase  =						[indices[0] for indices in total_indices]
              self.assertGreater(np.min(a__						)       ,     0						)
              self.assertListEqual([4, 3, 2, 1, 0]       ,     a__						)

       def 			__snake_case      ( self					:				Tuple						):
              import faiss

              UpperCAmelCase  =						FaissIndex(string_factory='''Flat'''						)
              index.add_vectors(np.eye(5       ,     dtype=np.floataa						)						)
              self.assertIsInstance(index.faiss_index       ,     faiss.IndexFlat						)
              UpperCAmelCase  =						FaissIndex(string_factory='''LSH'''						)
              index.add_vectors(np.eye(5       ,     dtype=np.floataa						)						)
              self.assertIsInstance(index.faiss_index       ,     faiss.IndexLSH						)
              with self.assertRaises(a__						):
                     UpperCAmelCase  =						FaissIndex(string_factory='''Flat'''       ,     custom_index=faiss.IndexFlat(5						)						)

       def 			__snake_case      ( self					:				List[Any]						):
              import faiss

              UpperCAmelCase  =						faiss.IndexFlat(5						)
              UpperCAmelCase  =						FaissIndex(custom_index=a__						)
              index.add_vectors(np.eye(5       ,     dtype=np.floataa						)						)
              self.assertIsInstance(index.faiss_index       ,     faiss.IndexFlat						)



       def 			__snake_case      ( self					:				Optional[Any]						):
              import faiss

              UpperCAmelCase  =						FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT						)
              index.add_vectors(np.eye(5       ,     dtype=np.floataa						)						)

              # Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
              # ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
              # see https://bugs.python.org/issue14243 and
              # https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
              with tempfile.NamedTemporaryFile(delete=a__						) as tmp_file:
                     index.save(tmp_file.name						)
                     UpperCAmelCase  =						FaissIndex.load(tmp_file.name						)
              os.unlink(tmp_file.name						)

              UpperCAmelCase  =						np.zeros(5       ,     dtype=np.floataa						)
              UpperCAmelCase  =						1
              UpperCAmelCase,					UpperCAmelCase  =						index.search(a__						)
              self.assertGreater(scores[0]       ,     0						)
              self.assertEqual(indices[0]       ,     1						)


@require_faiss
def       __snake_case		( SCREAMING_SNAKE_CASE_ :			Optional[int] )							->     Tuple:

       """simple docstring"""


       import faiss

       UpperCAmelCase  =						FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT )
       index.add_vectors(np.eye(5			,	dtype=np.floataa ) )

       UpperCAmelCase  =						'''index.faiss'''
       UpperCAmelCase  =						f"mock://{index_name}"
       index.save(SCREAMING_SNAKE_CASE_			,	storage_options=mockfs.storage_options )
       UpperCAmelCase  =						FaissIndex.load(SCREAMING_SNAKE_CASE_			,	storage_options=mockfs.storage_options )

       UpperCAmelCase  =						np.zeros(5			,	dtype=np.floataa )
       UpperCAmelCase  =						1
       UpperCAmelCase,					UpperCAmelCase  =						index.search(SCREAMING_SNAKE_CASE_ )
       assert scores[0] > 0
       assert indices[0] == 1






@require_elasticsearch
class     lowerCAmelCase__ (				UpperCAmelCase_    ):

       '''simple docstring'''

       def 			__snake_case      ( self					:				Tuple						):
              from elasticsearch import Elasticsearch

              with patch('''elasticsearch.Elasticsearch.search'''						) as mocked_search, patch(
                  '''elasticsearch.client.IndicesClient.create'''						) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''						) as mocked_bulk:
                     UpperCAmelCase  =						Elasticsearch()
                     UpperCAmelCase  =						{'''acknowledged''': True}
                     UpperCAmelCase  =						ElasticSearchIndex(es_client=a__						)
                     mocked_bulk.return_value([(True, None)] * 3						)
                     index.add_documents(['''foo''', '''bar''', '''foobar''']						)

                     # single query
                     UpperCAmelCase  =						'''foo'''
                     UpperCAmelCase  =						{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
                     UpperCAmelCase,					UpperCAmelCase  =						index.search(a__						)
                     self.assertEqual(scores[0]       ,     1						)
                     self.assertEqual(indices[0]       ,     0						)

                     # single query with timeout
                     UpperCAmelCase  =						'''foo'''
                     UpperCAmelCase  =						{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
                     UpperCAmelCase,					UpperCAmelCase  =						index.search(a__       ,     request_timeout=30						)
                     self.assertEqual(scores[0]       ,     1						)
                     self.assertEqual(indices[0]       ,     0						)

                     # batched queries
                     UpperCAmelCase  =						['''foo''', '''bar''', '''foobar''']
                     UpperCAmelCase  =						{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
                     UpperCAmelCase,					UpperCAmelCase  =						index.search_batch(a__						)
                     UpperCAmelCase  =						[scores[0] for scores in total_scores]
                     UpperCAmelCase  =						[indices[0] for indices in total_indices]
                     self.assertGreater(np.min(a__						)       ,     0						)
                     self.assertListEqual([1, 1, 1]       ,     a__						)

                     # batched queries with timeout
                     UpperCAmelCase  =						['''foo''', '''bar''', '''foobar''']
                     UpperCAmelCase  =						{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
                     UpperCAmelCase,					UpperCAmelCase  =						index.search_batch(a__       ,     request_timeout=30						)
                     UpperCAmelCase  =						[scores[0] for scores in total_scores]
                     UpperCAmelCase  =						[indices[0] for indices in total_indices]
                     self.assertGreater(np.min(a__						)       ,     0						)
                     self.assertListEqual([1, 1, 1]       ,     a__						)





from ...utils import logging
from ..ta.modeling_tf_ta import TFTaEncoderModel, TFTaForConditionalGeneration, TFTaModel
from .configuration_mta import MTaConfig


A									=  logging.get_logger(__name__)

A									=  'T5Config'


class SCREAMING_SNAKE_CASE	(		__snake_case				):






			"""simple docstring"""




			__A    	=	"""mt5"""
			__A    	=	MTaConfig







class SCREAMING_SNAKE_CASE	(		__snake_case				):






			"""simple docstring"""




			__A    	=	"""mt5"""
			__A    	=	MTaConfig







class SCREAMING_SNAKE_CASE	(		__snake_case				):






			"""simple docstring"""




			__A    	=	"""mt5"""
			__A    	=	MTaConfig
